Spaces:
Runtime error
Runtime error
File size: 10,667 Bytes
a84678a cb05f51 a84678a cb05f51 a84678a cb05f51 a84678a 8a998bc a84678a 8a998bc a84678a 8a998bc cb05f51 a84678a 8a998bc 35a28f9 a84678a 8a998bc a84678a cb05f51 a84678a cb05f51 a84678a cb05f51 a84678a 8a998bc a84678a 8a998bc a84678a 5d9145f 88b0888 5d9145f a84678a 88b0888 a84678a 5d9145f 0c51c37 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 |
import evaluate
import datasets
import numpy as np
# TODO: Add BibTeX citation
_CITATION = """\
@InProceedings{huggingface:module,
title = {A great new module},
authors={huggingface, Inc.},
year={2020}
}
"""
# TODO: Add description of the module here
_DESCRIPTION = """\
This metric is used for evaluating the quality of relation extraction output. By calculating the Micro and Macro F1 score of every relation extraction outputs to ensure the quality.
"""
_KWARGS_DESCRIPTION = """
Calculates how good are predictions given some references, using Precision, Recall, F1 Score.
Args:
predictions (list of list of dictionary): A list of predicted relations from the model.
references (list of list of dictionary): A list of ground-truth or reference relations to compare the predictions against.
Returns:
**output** (`dictionary` of `dictionary`s) A dictionary mapping each entity type to its respective scoring metrics such as Precision, Recall, F1 Score.
- **entity type** (`dictionary`): score of selected relation type
- **tp** : true positive count
- **fp** : false positive count
- **fn** : false negative count
- **p** : precision
- **r** : recall
- **f1** : micro f1 score
- **Macro_f1** : macro f1 score
- **Macro_p** : macro precision
- **Macro_r** : macro recall
Examples:
metric_path = "Ikala-allen/relation_extraction"
module = evaluate.load(metric_path)
references = [
[
{"head": "phipigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
{"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
{'head': 'A醛賦活緊緻精華', 'tail': 'Serum', 'head_type': 'product', 'tail_type': 'category', 'type': 'belongs_to'},
]
]
predictions = [
[
{"head": "phipigments", "head_type": "product", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
{"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
]
]
evaluation_scores = module.compute(predictions=predictions, references=references, mode="strict", detailed_scores=False, relation_types=[])
print(evaluation_scores)
{'tp': 1, 'fp': 1, 'fn': 2, 'p': 50.0, 'r': 33.333333333333336, 'f1': 40.0, 'Macro_f1': 25.0, 'Macro_p': 25.0, 'Macro_r': 25.0}
"""
def convert_format(data:list):
"""
Args:
data (list) : list of dictionaries with different entity elements
e.g
[
{'head': ['phipigments', 'tinadaviespigments'...],
'head_type': ['product', 'brand'...],
'type': ['sell', 'sell'...],
'tail': ['國際認證之色乳', '國際認證之色乳'...],
'tail_type': ['product', 'product'...]},
{'head': ['SABONTAIWAN', 'SNTAIWAN'...],
'head_type': ['brand', 'brand'...],
'type': ['sell', 'sell'...],
'tail': ['大馬士革玫瑰有機光燦系列', '大馬士革玫瑰有機光燦系列'...],
'tail_type': ['product', 'product'...]}
...
]
"""
predictions = []
for item in data:
prediction_group = []
for i in range(len(item['head'])):
prediction = {
'head': item['head'][i],
'head_type': item['head_type'][i],
'type': item['type'][i],
'tail': item['tail'][i],
'tail_type': item['tail_type'][i],
}
prediction_group.append(prediction)
predictions.append(prediction_group)
return predictions
@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
class relation_extraction(evaluate.Metric):
"""evaluating the quality of relation extraction output"""
def _info(self):
# TODO: Specifies the evaluate.EvaluationModuleInfo object
return evaluate.MetricInfo(
# This is the description that will appear on the modules page.
module_type="metric",
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
# This defines the format of each prediction and reference
features=datasets.Features({
'predictions': datasets.Sequence({
"head": datasets.Value("string"),
"head_type": datasets.Value("string"),
"type": datasets.Value("string"),
"tail": datasets.Value("string"),
"tail_type": datasets.Value("string"),
}),
'references': datasets.Sequence({
"head": datasets.Value("string"),
"head_type": datasets.Value("string"),
"type": datasets.Value("string"),
"tail": datasets.Value("string"),
"tail_type": datasets.Value("string"),
}),
}),
# Homepage of the module for documentation
homepage="http://module.homepage",
# Additional links to the codebase or references
codebase_urls=["http://github.com/path/to/codebase/of/new_module"],
reference_urls=["http://path.to.reference.url/new_module"]
)
def _download_and_prepare(self, dl_manager):
pass
def _compute(self, predictions, references, mode, detailed_scores=False, relation_types=[]):
"""
This method computes and returns various scoring metrics for the prediction model based on the mode specified, including Precision, Recall, F1-Score and others. It evaluates the model's predictions against the provided reference data.
Parameters:
predictions: A list of predicted relations from the model.
references: A list of ground-truth or reference relations to compare the predictions against.
mode: Evaluation mode - 'strict' or 'boundaries'. 'strict' mode takes into account both entities type and their relationships
while 'boundaries' mode only considers the entity spans of the relationships.
detailed_scores: Boolean value, if True it returns scores for each relation type specifically,
if False it returns the overall scores.
relation_types: A list of relation types to consider while evaluating. If not provided, relation types will be constructed
from the ground truth or reference data.
Returns:
A dictionary mapping each entity type to its respective scoring metrics such as Precision, Recall, F1 Score.
"""
predictions = convert_format(predictions)
references = convert_format(references)
assert mode in ["strict", "boundaries"]
# construct relation_types from ground truth if not given
if len(relation_types) == 0:
for triplets in references:
for triplet in triplets:
relation = triplet["type"]
if relation not in relation_types:
relation_types.append(relation)
scores = {rel: {"tp": 0, "fp": 0, "fn": 0} for rel in relation_types + ["ALL"]}
# Count GT relations and Predicted relations
n_sents = len(references)
n_rels = sum([len([rel for rel in sent]) for sent in references])
n_found = sum([len([rel for rel in sent]) for sent in predictions])
# Count TP, FP and FN per type
for pred_sent, gt_sent in zip(predictions, references):
for rel_type in relation_types:
# strict mode takes argument types into account
if mode == "strict":
pred_rels = {(rel["head"], rel["head_type"], rel["tail"], rel["tail_type"]) for rel in pred_sent if
rel["type"] == rel_type}
gt_rels = {(rel["head"], rel["head_type"], rel["tail"], rel["tail_type"]) for rel in gt_sent if
rel["type"] == rel_type}
# boundaries mode only takes argument spans into account
elif mode == "boundaries":
pred_rels = {(rel["head"], rel["tail"]) for rel in pred_sent if rel["type"] == rel_type}
gt_rels = {(rel["head"], rel["tail"]) for rel in gt_sent if rel["type"] == rel_type}
scores[rel_type]["tp"] += len(pred_rels & gt_rels)
scores[rel_type]["fp"] += len(pred_rels - gt_rels)
scores[rel_type]["fn"] += len(gt_rels - pred_rels)
# Compute per entity Precision / Recall / F1
for rel_type in scores.keys():
if scores[rel_type]["tp"]:
scores[rel_type]["p"] = 100 * scores[rel_type]["tp"] / (scores[rel_type]["fp"] + scores[rel_type]["tp"])
scores[rel_type]["r"] = 100 * scores[rel_type]["tp"] / (scores[rel_type]["fn"] + scores[rel_type]["tp"])
else:
scores[rel_type]["p"], scores[rel_type]["r"] = 0, 0
if not scores[rel_type]["p"] + scores[rel_type]["r"] == 0:
scores[rel_type]["f1"] = 2 * scores[rel_type]["p"] * scores[rel_type]["r"] / (
scores[rel_type]["p"] + scores[rel_type]["r"])
else:
scores[rel_type]["f1"] = 0
# Compute micro F1 Scores
tp = sum([scores[rel_type]["tp"] for rel_type in relation_types])
fp = sum([scores[rel_type]["fp"] for rel_type in relation_types])
fn = sum([scores[rel_type]["fn"] for rel_type in relation_types])
if tp:
precision = 100 * tp / (tp + fp)
recall = 100 * tp / (tp + fn)
f1 = 2 * precision * recall / (precision + recall)
else:
precision, recall, f1 = 0, 0, 0
scores["ALL"]["p"] = precision
scores["ALL"]["r"] = recall
scores["ALL"]["f1"] = f1
scores["ALL"]["tp"] = tp
scores["ALL"]["fp"] = fp
scores["ALL"]["fn"] = fn
# Compute Macro F1 Scores
scores["ALL"]["Macro_f1"] = np.mean([scores[ent_type]["f1"] for ent_type in relation_types])
scores["ALL"]["Macro_p"] = np.mean([scores[ent_type]["p"] for ent_type in relation_types])
scores["ALL"]["Macro_r"] = np.mean([scores[ent_type]["r"] for ent_type in relation_types])
if detailed_scores:
return scores
return scores["ALL"] |