Update README.md

README.md

@@ -20,7 +20,7 @@ This metric is used for evaluating the quality of relation extraction output. By
 
 
 ## Metric Description
-This metric can be used in relation extraction evaluation. 
+This metric can be used in relation extraction evaluation.
 
 ## How to Use
 This metric takes 2 inputs, prediction and references(ground truth). Both of them are a list of list of dictionary of entity's name and entity's type:
@@ -79,7 +79,7 @@ Output Example:
 Remind : Macro_f1、Macro_p、Macro_r、p、r、f1 are always a number between 0 and 1. And tp、fp、fn depend on how many data inputs.
 
 ### Examples
-Example1 : only one prediction and reference, mode = strict,  only output ALL relation score
+Example1 : only one prediction and reference, mode = strict, only output ALL relation score
 ```python
 metric_path = "Ikala-allen/relation_extraction"
 module = evaluate.load(metric_path)
@@ -133,7 +133,7 @@ print(evaluation_scores)
 >>> {'tp': 2, 'fp': 0, 'fn': 1, 'p': 100.0, 'r': 66.66666666666667, 'f1': 80.0, 'Macro_f1': 50.0, 'Macro_p': 50.0, 'Macro_r': 50.0}
 ```
 
-Example3 :  two or more prediction and reference, mode = boundaries, only output = False, output all relation type
+Example3 : two or more prediction and reference, mode = boundaries, only output = False, output all relation type score
 ```python
 metric_path = "Ikala-allen/relation_extraction"
 module = evaluate.load(metric_path)
@@ -168,34 +168,40 @@ print(evaluation_scores)
 >>> {'sell': {'tp': 3, 'fp': 1, 'fn': 0, 'p': 75.0, 'r': 100.0, 'f1': 85.71428571428571}, 'belongs_to': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0}, 'ALL': {'tp': 3, 'fp': 1, 'fn': 1, 'p': 75.0, 'r': 75.0, 'f1': 75.0, 'Macro_f1': 42.857142857142854, 'Macro_p': 37.5, 'Macro_r': 50.0}}
 ```
 
-Example 4 with two or more prediction and reference:
+Example 4 : two or more prediction and reference, mode = boundaries, only output = False, only output ALL relation score, relation_types  = ["belongs_to"], only consider belongs_to type score
 ```python
->>> metric_path = "Ikala-allen/relation_extraction"
->>> module = evaluate.load(metric_path)
->>> references = [
-...     [
-...         {"head": "phip igments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
-...         {"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
-...     ],[
-...           {'head': 'SABONTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'}
-...     ]
-...   ]
->>> predictions = [
-...    [
-...        {"head": "phipigments", "head_type": "product", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
-...        {"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
-...    ],[
-...          {'head': 'SABONTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'},
-...          {'head': 'SNTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'}  
-...    ]
-...  ]
->>> evaluation_scores = module.compute(predictions=predictions, references=references)  
->>> print(evaluation_scores)  
-{'sell': {'tp': 2, 'fp': 2, 'fn': 1, 'p': 50.0, 'r': 66.66666666666667, 'f1': 57.142857142857146}, 'ALL': {'tp': 2, 'fp': 2, 'fn': 1, 'p': 50.0, 'r': 66.66666666666667, 'f1': 57.142857142857146, 'Macro_f1': 57.142857142857146, 'Macro_p': 50.0, 'Macro_r': 66.66666666666667}}
+metric_path = "Ikala-allen/relation_extraction"
+module = evaluate.load(metric_path)
+references = [
+  [
+    {"head": "phipigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
+    {"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
+  ],
+  [
+    {'head': 'SABONTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'},
+    {'head': 'A醛賦活緊緻精華', 'tail': 'Serum', 'head_type': 'product', 'tail_type': 'category', 'type': 'belongs_to'},
+  ]
+]
+
+# Example references (ground truth)
+predictions = [
+  [
+    {"head": "phipigments", "head_type": "product", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
+    {"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
+  ],
+  [
+    {'head': 'SABONTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'},
+    {'head': 'SNTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'}  
+  ]
+]
+evaluation_scores = module.compute(predictions=predictions, references=references, mode = "boundaries", only_all=False,relation_types  = ["belongs_to"])
+print(evaluation_scores)  
+>>> {'belongs_to': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0}, 'ALL': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0, 'Macro_f1': 0.0, 'Macro_p': 0.0, 'Macro_r': 0.0}}
 ```
 
 ## Limitations and Bias
-This metric has strict filter mechanism, if any of the prediction's entity names, such as head, head_type, type, tail, or tail_type, is not exactly the same as the reference one. It will count as fp or fn. 
+This metric has strict and boundaries mode, also can select relation_types for different type evaluation. Make sure to select suitable evaluation parameters. F1 score may be totally different.
+Prediction and reference entity_name should be exactly the same regardless of case and spaces. If prediction is not exactly the same as the reference one. It will count as fp or fn. 
 
 ## Citation
 ```bibtex