Ikala-allen commited on
Commit
65757ca
1 Parent(s): 0c51c37

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +10 -10
README.md CHANGED
@@ -47,7 +47,7 @@ evaluation_scores = module.compute(predictions=predictions, references=reference
47
  - **predictions** (`list` of `list` of `dictionary`): a list of list of dictionary with every element's relation_type and their entity name
48
  - **references** (`list` of `list` of `dictionary`): a list of list of dictionary with every element's relation_type and their entity name
49
  - **mode** (`str`): define strict or boundaries mode for evaluation, strict mode consider "head_type" and "tail_type", boundaries mode doesn't consider "head_type" and "tail_type"
50
- - **only** (`bool`): True for only output all relation_type score. False for output every relation_type score, default True
51
  - **relation_types** (`list`): define which relation type that need to be evaluate and show, if not given, it will construct relation_types from ground truth, default []
52
 
53
  ### Output Values
@@ -79,7 +79,7 @@ Output Example:
79
  Remind : Macro_f1、Macro_p、Macro_r、p、r、f1 are always a number between 0 and 1. And tp、fp、fn depend on how many data inputs.
80
 
81
  ### Examples
82
- Example1 : only one prediction and reference, mode=strict, only output total relation score
83
  ```python
84
  metric_path = "Ikala-allen/relation_extraction"
85
  module = evaluate.load(metric_path)
@@ -96,12 +96,12 @@ predictions = [
96
  {"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
97
  ]
98
  ]
99
- evaluation_scores = module.compute(predictions=predictions, references=references, mode="strict", only=True, relation_types=[])
100
  print(evaluation_scores)
101
  >>> {'tp': 1, 'fp': 1, 'fn': 2, 'p': 50.0, 'r': 33.333333333333336, 'f1': 40.0, 'Macro_f1': 25.0, 'Macro_p': 25.0, 'Macro_r': 25.0}
102
  ```
103
 
104
- Example2 : only one prediction and reference, mode=boundaries, only output total relation score
105
  ```python
106
  metric_path = "Ikala-allen/relation_extraction"
107
  module = evaluate.load(metric_path)
@@ -118,12 +118,12 @@ predictions = [
118
  {"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
119
  ]
120
  ]
121
- evaluation_scores = module.compute(predictions=predictions, references=references, mode="strict", only=True, relation_types=[])
122
  print(evaluation_scores)
123
- >>> {'tp': 2, 'fp': 0, 'fn': 1, 'p': 100.0, 'r': 66.66666666666667, 'f1': 80.0, 'Macro_f1': 50.0, 'Macro_p': 50.0, 'Macro_r': 50.0}
124
  ```
125
 
126
- Example3 : two or more prediction and reference, mode=boundaries, only=False, output all relation type score
127
  ```python
128
  metric_path = "Ikala-allen/relation_extraction"
129
  module = evaluate.load(metric_path)
@@ -147,12 +147,12 @@ predictions = [
147
  {'head': 'SNTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'}
148
  ]
149
  ]
150
- evaluation_scores = module.compute(predictions=predictions, references=references, mode="boundaries", only=False, relation_types=[])
151
  print(evaluation_scores)
152
  >>> {'sell': {'tp': 3, 'fp': 1, 'fn': 0, 'p': 75.0, 'r': 100.0, 'f1': 85.71428571428571}, 'belongs_to': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0}, 'ALL': {'tp': 3, 'fp': 1, 'fn': 1, 'p': 75.0, 'r': 75.0, 'f1': 75.0, 'Macro_f1': 42.857142857142854, 'Macro_p': 37.5, 'Macro_r': 50.0}}
153
  ```
154
 
155
- Example 4 : two or more prediction and reference, mode=boundaries, only=False, only output total relation score, relation_types = ["belongs_to"], only consider belongs_to type score
156
  ```python
157
  metric_path = "Ikala-allen/relation_extraction"
158
  module = evaluate.load(metric_path)
@@ -176,7 +176,7 @@ predictions = [
176
  {'head': 'SNTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'}
177
  ]
178
  ]
179
- evaluation_scores = module.compute(predictions=predictions, references=references, mode="boundaries", only=False, relation_types=["belongs_to"])
180
  print(evaluation_scores)
181
  >>> {'belongs_to': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0}, 'ALL': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0, 'Macro_f1': 0.0, 'Macro_p': 0.0, 'Macro_r': 0.0}}
182
  ```
 
47
  - **predictions** (`list` of `list` of `dictionary`): a list of list of dictionary with every element's relation_type and their entity name
48
  - **references** (`list` of `list` of `dictionary`): a list of list of dictionary with every element's relation_type and their entity name
49
  - **mode** (`str`): define strict or boundaries mode for evaluation, strict mode consider "head_type" and "tail_type", boundaries mode doesn't consider "head_type" and "tail_type"
50
+ - **show** (`bool`): False for only output all relation_type score. True for output every relation_type score, default False
51
  - **relation_types** (`list`): define which relation type that need to be evaluate and show, if not given, it will construct relation_types from ground truth, default []
52
 
53
  ### Output Values
 
79
  Remind : Macro_f1、Macro_p、Macro_r、p、r、f1 are always a number between 0 and 1. And tp、fp、fn depend on how many data inputs.
80
 
81
  ### Examples
82
+ Example1 : only one prediction and reference, mode=strict, show=False, only output total relation score
83
  ```python
84
  metric_path = "Ikala-allen/relation_extraction"
85
  module = evaluate.load(metric_path)
 
96
  {"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
97
  ]
98
  ]
99
+ evaluation_scores = module.compute(predictions=predictions, references=references, mode="strict", show=False, relation_types=[])
100
  print(evaluation_scores)
101
  >>> {'tp': 1, 'fp': 1, 'fn': 2, 'p': 50.0, 'r': 33.333333333333336, 'f1': 40.0, 'Macro_f1': 25.0, 'Macro_p': 25.0, 'Macro_r': 25.0}
102
  ```
103
 
104
+ Example2 : only one prediction and reference, mode=boundaries, show=False, only output total relation score
105
  ```python
106
  metric_path = "Ikala-allen/relation_extraction"
107
  module = evaluate.load(metric_path)
 
118
  {"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
119
  ]
120
  ]
121
+ evaluation_scores = module.compute(predictions=predictions, references=references, mode="strict", show=False, relation_types=[])
122
  print(evaluation_scores)
123
+ >>> {'tp': 1, 'fp': 1, 'fn': 2, 'p': 50.0, 'r': 33.333333333333336, 'f1': 40.0, 'Macro_f1': 25.0, 'Macro_p': 25.0, 'Macro_r': 25.0}
124
  ```
125
 
126
+ Example3 : two or more prediction and reference, mode=boundaries, show=True, output all relation type score
127
  ```python
128
  metric_path = "Ikala-allen/relation_extraction"
129
  module = evaluate.load(metric_path)
 
147
  {'head': 'SNTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'}
148
  ]
149
  ]
150
+ evaluation_scores = module.compute(predictions=predictions, references=references, mode="boundaries", show=True, relation_types=[])
151
  print(evaluation_scores)
152
  >>> {'sell': {'tp': 3, 'fp': 1, 'fn': 0, 'p': 75.0, 'r': 100.0, 'f1': 85.71428571428571}, 'belongs_to': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0}, 'ALL': {'tp': 3, 'fp': 1, 'fn': 1, 'p': 75.0, 'r': 75.0, 'f1': 75.0, 'Macro_f1': 42.857142857142854, 'Macro_p': 37.5, 'Macro_r': 50.0}}
153
  ```
154
 
155
+ Example 4 : two or more prediction and reference, mode=boundaries, show=True, output all relation type score, relation_types = ["belongs_to"], only consider belongs_to type score
156
  ```python
157
  metric_path = "Ikala-allen/relation_extraction"
158
  module = evaluate.load(metric_path)
 
176
  {'head': 'SNTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'}
177
  ]
178
  ]
179
+ evaluation_scores = module.compute(predictions=predictions, references=references, mode="boundaries", show=True, relation_types=["belongs_to"])
180
  print(evaluation_scores)
181
  >>> {'belongs_to': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0}, 'ALL': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0, 'Macro_f1': 0.0, 'Macro_p': 0.0, 'Macro_r': 0.0}}
182
  ```