Spaces:
Build error
Build error
File size: 6,298 Bytes
20e9c0d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 |
import six
def check_predicted_evidence_format(instance):
if 'predicted_evidence' in instance.keys() and len(instance['predicted_evidence']):
assert all(isinstance(prediction, list)
for prediction in instance["predicted_evidence"]), \
"Predicted evidence must be a list of (page,line) lists"
assert all(len(prediction) == 2
for prediction in instance["predicted_evidence"]), \
"Predicted evidence must be a list of (page,line) lists"
assert all(isinstance(prediction[0], six.string_types)
for prediction in instance["predicted_evidence"]), \
"Predicted evidence must be a list of (page<string>,line<int>) lists"
assert all(isinstance(prediction[1], int)
for prediction in instance["predicted_evidence"]), \
"Predicted evidence must be a list of (page<string>,line<int>) lists"
def is_correct_label(instance):
return instance["label"].upper() == instance["predicted_label"].upper()
def is_strictly_correct(instance, max_evidence=None):
#Strict evidence matching is only for NEI class
check_predicted_evidence_format(instance)
if instance["label"].upper() != "NOT ENOUGH INFO" and is_correct_label(instance):
assert 'predicted_evidence' in instance, "Predicted evidence must be provided for strict scoring"
if max_evidence is None:
max_evidence = len(instance["predicted_evidence"])
for evience_group in instance["evidence"]:
#Filter out the annotation ids. We just want the evidence page and line number
actual_sentences = [[e[2], e[3]] for e in evience_group]
#Only return true if an entire group of actual sentences is in the predicted sentences
if all([actual_sent in instance["predicted_evidence"][:max_evidence] for actual_sent in actual_sentences]):
return True
#If the class is NEI, we don't score the evidence retrieval component
elif instance["label"].upper() == "NOT ENOUGH INFO" and is_correct_label(instance):
return True
return False
def evidence_macro_precision(instance, max_evidence=None):
this_precision = 0.0
this_precision_hits = 0.0
if instance["label"].upper() != "NOT ENOUGH INFO":
all_evi = [[e[2], e[3]] for eg in instance["evidence"] for e in eg if e[3] is not None]
predicted_evidence = instance["predicted_evidence"] if max_evidence is None else \
instance["predicted_evidence"][:max_evidence]
for prediction in predicted_evidence:
if prediction in all_evi:
this_precision += 1.0
this_precision_hits += 1.0
return (this_precision / this_precision_hits) if this_precision_hits > 0 else 1.0, 1.0
return 0.0, 0.0
def evidence_macro_recall(instance, max_evidence=None):
# We only want to score F1/Precision/Recall of recalled evidence for NEI claims
if instance["label"].upper() != "NOT ENOUGH INFO":
# If there's no evidence to predict, return 1
if len(instance["evidence"]) == 0 or all([len(eg) == 0 for eg in instance]):
return 1.0, 1.0
predicted_evidence = instance["predicted_evidence"] if max_evidence is None else \
instance["predicted_evidence"][:max_evidence]
for evidence_group in instance["evidence"]:
evidence = [[e[2], e[3]] for e in evidence_group]
if all([item in predicted_evidence for item in evidence]):
# We only want to score complete groups of evidence. Incomplete groups are worthless.
return 1.0, 1.0
return 0.0, 1.0
return 0.0, 0.0
# Micro is not used. This code is just included to demostrate our model of macro/micro
def evidence_micro_precision(instance):
this_precision = 0
this_precision_hits = 0
# We only want to score Macro F1/Precision/Recall of recalled evidence for NEI claims
if instance["label"].upper() != "NOT ENOUGH INFO":
all_evi = [[e[2], e[3]] for eg in instance["evidence"] for e in eg if e[3] is not None]
for prediction in instance["predicted_evidence"]:
if prediction in all_evi:
this_precision += 1.0
this_precision_hits += 1.0
return this_precision, this_precision_hits
def fever_score(predictions,actual=None, max_evidence=5):
correct = 0
strict = 0
macro_precision = 0
macro_precision_hits = 0
macro_recall = 0
macro_recall_hits = 0
for idx,instance in enumerate(predictions):
assert 'predicted_evidence' in instance.keys(), 'evidence must be provided for the prediction'
#If it's a blind test set, we need to copy in the values from the actual data
if 'evidence' not in instance or 'label' not in instance:
assert actual is not None, 'in blind evaluation mode, actual data must be provided'
assert len(actual) == len(predictions), 'actual data and predicted data length must match'
assert 'evidence' in actual[idx].keys(), 'evidence must be provided for the actual evidence'
instance['evidence'] = actual[idx]['evidence']
instance['label'] = actual[idx]['label']
assert 'evidence' in instance.keys(), 'gold evidence must be provided'
if is_correct_label(instance):
correct += 1.0
if is_strictly_correct(instance, max_evidence):
strict+=1.0
macro_prec = evidence_macro_precision(instance, max_evidence)
macro_precision += macro_prec[0]
macro_precision_hits += macro_prec[1]
macro_rec = evidence_macro_recall(instance, max_evidence)
macro_recall += macro_rec[0]
macro_recall_hits += macro_rec[1]
total = len(predictions)
strict_score = strict / total
acc_score = correct / total
pr = (macro_precision / macro_precision_hits) if macro_precision_hits > 0 else 1.0
rec = (macro_recall / macro_recall_hits) if macro_recall_hits > 0 else 0.0
f1 = 2.0 * pr * rec / (pr + rec)
return strict_score, acc_score, pr, rec, f1 |