File size: 4,447 Bytes
2d8da09 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 |
# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import json
import re
import string
from collections import Counter
from rouge_score import rouge_scorer
"""
This script can be used to calcualte exact match and F1 scores for many different tasks.
The file "squad_test_predictions.jsonl" is assumed to be generated by the
`examples/nlp/language_modeling/tuning/megatron_gpt_peft_eval.py` script
Example command for GPT Preds
```
python peft_metric_calc.py \
--pred_file squad_test_predictions.jsonl \
--label_field "original_answers" \
```
"""
def normalize_answer(s):
"""Lower text and remove punctuation, articles and extra whitespace."""
def remove_articles(text):
return re.sub(r'\b(a|an|the)\b', ' ', text)
def white_space_fix(text):
return ' '.join(text.split())
def remove_punc(text):
exclude = set(string.punctuation)
return ''.join(ch for ch in text if ch not in exclude)
def lower(text):
return text.lower()
return white_space_fix(remove_articles(remove_punc(lower(s))))
def f1_score(prediction, ground_truth):
prediction_tokens = normalize_answer(prediction).split()
ground_truth_tokens = normalize_answer(ground_truth).split()
common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
num_same = sum(common.values())
if num_same == 0:
return 0
precision = 1.0 * num_same / len(prediction_tokens)
recall = 1.0 * num_same / len(ground_truth_tokens)
f1 = (2 * precision * recall) / (precision + recall)
return f1
def exact_match_score(prediction, ground_truth):
return normalize_answer(prediction) == normalize_answer(ground_truth)
def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
scores_for_ground_truths = []
for ground_truth in ground_truths:
score = metric_fn(prediction, ground_truth)
scores_for_ground_truths.append(score)
return max(scores_for_ground_truths)
def main():
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument(
'--pred_file',
type=str,
help="Text file with test set prompts + model predictions. Prediction file can be made by running NeMo/examples/nlp/language_modeling/megatron_gpt_prompt_learning_eval.py",
)
parser.add_argument(
'--pred_field',
type=str,
help="The field in the json file that contains the prediction tokens",
default="pred",
)
parser.add_argument(
'--label_field',
type=str,
help="The field in the json file that contains the ground truth tokens",
default="label",
)
args = parser.parse_args()
pred_file = args.pred_file
scorer = rouge_scorer.RougeScorer(['rougeL'], use_stemmer=True)
preds = open(pred_file, encoding="utf-8").readlines()
f1 = exact_match = total = r_score = 0
for i in range(len(preds)):
pred_line = json.loads(preds[i])
pred_answer = pred_line[args.pred_field]
true_answers = pred_line[args.label_field]
if not isinstance(true_answers, list):
true_answers = [true_answers]
r_scores = []
for ta in true_answers:
r_scores.append(scorer.score(ta, pred_answer)['rougeL'].fmeasure)
r_score += max(r_scores)
exact_match += metric_max_over_ground_truths(exact_match_score, pred_answer, true_answers)
f1 += metric_max_over_ground_truths(f1_score, pred_answer, true_answers)
total += 1
exact_match = 100.0 * exact_match / total
f1 = 100.0 * f1 / total
r_score = 100 * (r_score / total)
res = {'exact_match': exact_match, 'f1': f1, "rougeL": r_score, 'total': total}
print('\t'.join([f"{k} {v:.3f}" for k, v in res.items()]))
if __name__ == "__main__":
main()
|