add

README.md

@@ -135,8 +135,8 @@ This metric has strict filter mechanism, if any of the prediction's entity names
     author = {Bruno Taillé, Vincent Guigue, Geoffrey Scoutheeten, Patrick Gallinari},
     title = {Let's Stop Incorrect Comparisons in End-to-end Relation Extraction!},
     year = {2020},
+    link = https://arxiv.org/abs/2009.10684
 }
-*https://arxiv.org/abs/2009.10684*
 ```
 ## Further References
 This evaluation metric implementation uses 

relation_extraction.py

@@ -2,45 +2,69 @@ import evaluate
 import datasets
 import numpy as np
 
-# TODO: Add BibTeX citation
+# Add BibTeX citation
 _CITATION = """\
-@InProceedings{huggingface:module,
-title = {A great new module},
-authors={huggingface, Inc.},
-year={2020}
+@Paper{
+    author = {Bruno Taillé, Vincent Guigue, Geoffrey Scoutheeten, Patrick Gallinari},
+    title = {Let's Stop Incorrect Comparisons in End-to-end Relation Extraction!},
+    year = {2020},
+    link = https://arxiv.org/abs/2009.10684
 }
 """
 
-# TODO: Add description of the module here
+# Add description of the module here
 _DESCRIPTION = """\
-This new module is designed to solve this great ML task and is crafted with a lot of care.
+This metric is used for evaluating the quality of relation extraction output. By calculating the Micro and Macro F1 score of every relation extraction outputs to ensure the quality.
 """
 
 
-# TODO: Add description of the arguments of the module here
+# Add description of the arguments of the module here
 _KWARGS_DESCRIPTION = """
-Calculates how good are predictions given some references, using certain scores
+Calculates how good are predictions given some references, using Micro and Macro F1 scores
 Args:
-    predictions: list of predictions to score. Each predictions
-        should be a string with tokens separated by spaces.
-    references: list of reference for each prediction. Each
-        reference should be a string with tokens separated by spaces.
+    predictions: list of list of dictionary, including relation and its type 
+        dictionary should be key value pair like entity name link to its type
+    references: list of list of dictionary, including relation and its type 
+        dictionary should be entity name pair like entity name link to its type
 Returns:
-    accuracy: description of the first score,
-    another_score: description of the second score,
+    evaluation result: 
+    - **sell** (`dictionary`): score of type sell
+  - **tp** : true positive count
+  - **fp** : false positive count
+  - **fn** : false negative count
+  - **p** : precision
+  - **r** : recall
+  - **f1** : micro f1 score
+- **ALL** (`dictionary`): score of all of the type (sell and belongs to)
+  - **tp** : true positive count
+  - **fp** : false positive count
+  - **fn** : false negative count
+  - **p** : precision
+  - **r** : recall
+  - **f1** : micro f1 score
+  - **Macro_f1** : macro f1 score
+  - **Macro_p** : macro precision
+  - **Macro_r** : macro recall
 Examples:
-    Examples should be written in doctest format, and should illustrate how
-    to use the function.
-    >>> my_new_module = evaluate.load("my_new_module")
-    >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1])
-    >>> print(results)
-    {'accuracy': 1.0}
+    >>> metric_path = "Ikala-allen/relation_extraction"
+    >>> module = evaluate.load(metric_path)
+    >>> references = [
+    ...    [
+    ...       {"head": "phip igments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
+    ...        {"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
+    ...    ]
+    ... ]
+    >>> predictions = [
+    ...    [
+    ...        {"head": "phipigments", "head_type": "product", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
+    ...        {"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
+    ...    ]
+    ... ]
+    >>> evaluation_scores = module.compute(predictions=predictions, references=references)
+    >>> print(evaluation_scores)
+    {'sell': {'tp': 1, 'fp': 1, 'fn': 1, 'p': 50.0, 'r': 50.0, 'f1': 50.0}, 'ALL': {'tp': 1, 'fp': 1, 'fn': 1, 'p': 50.0, 'r': 50.0, 'f1': 50.0, 'Macro_f1': 50.0, 'Macro_p': 50.0, 'Macro_r': 50.0}}
 """
 
-# TODO: Define external resources urls if needed
-BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
-
-
 def convert_format(data:list):
     """
     Args:
@@ -51,13 +75,12 @@ def convert_format(data:list):
         'head_type': ['product', 'brand'...], 
         'type': ['sell', 'sell'...], 
         'tail': ['國際認證之色乳', '國際認證之色乳'...], 
-        'tail_type': ['product', 'product'...]}, 
-
+        'tail_type': ['product', 'product'...]},  # first element
         {'head': ['SABONTAIWAN', 'SNTAIWAN'...], 
         'head_type': ['brand', 'brand'...], 
         'type': ['sell', 'sell'...], 
         'tail': ['大馬士革玫瑰有機光燦系列', '大馬士革玫瑰有機光燦系列'...], 
-        'tail_type': ['product', 'product'...]}
+        'tail_type': ['product', 'product'...]}  # second element
         ...
         ]
     """
@@ -78,7 +101,12 @@ def convert_format(data:list):
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class relation_extraction(evaluate.Metric):
-    """TODO: Short description of my evaluation module."""
+    """
+    evaluation metric of relation extraction
+    inputs:
+        predictions : (`list` of `list`s of `dictionary`s) about relation and its type of prediction
+        references : (`list` of `list`s of `dictionary`s) about references for each relation and its type.
+    """
 
     def _info(self):
         # TODO: Specifies the evaluate.EvaluationModuleInfo object