Upload eval_utils.py

eval_utils.py

@@ -3,157 +3,19 @@ import re
 from collections import defaultdict
 
 import evaluate
-
-# import nltk
+import nltk
 import numpy as np
 from nervaluate import Evaluator
-from rouge_score import rouge_scorer
+# from rouge_score import rouge_scorer
 from sacrebleu.metrics import BLEU, CHRF
 from sklearn.metrics import f1_score
 from tqdm import tqdm
 from transformers import AutoTokenizer
-
-from transformers import AutoTokenizer
-import re
+import rouge
+import bert_score
 import string
 
-
-class TF_Tokenizer:
-    def __init__(self, model_str):
-        tok = AutoTokenizer.from_pretrained(model_str)
-
-    def __call__(self, txt):
-        return self.tok.tokenize(txt)
-
-
-class WS_Tokenizer:
-    def __init__(self):
-        pass
-
-    def __call__(self, txt):
-        return re.findall(r"[{}]|\w+".format(string.punctuation), txt)
-
-
-def convert_spans_to_bio(txt, roles, tokenizer_func):
-    roles = sorted(roles, key=lambda x: x["start"])
-    roles_left = [r["start"] for r in roles]
-
-    ttxt = tokenizer_func(txt)
-
-    c = 0
-    cr = -1
-    prev = "O"
-    troles = []
-    for tok in ttxt:
-        if c >= len(txt):
-            break
-
-        while txt[c] == " ":
-            c += 1
-
-        else:
-            if c in roles_left:  # Start of a new role
-                ind = roles_left.index(c)
-                cr = roles[ind]["end"]
-                prev = "I-" + roles[ind]["label"]
-                troles.append("B-" + roles[ind]["label"])
-            else:
-                if c < cr:  # Assign previous role
-                    troles.append(prev)
-                else:  # Assign 'O'
-                    troles.append("O")
-
-            c += len(tok)
-
-    if len(ttxt) != len(troles):
-        troles += ["O"] * (len(ttxt) - len(troles))
-
-    assert len(ttxt) == len(troles)
-    return troles
-
-
-def convert_bio_to_spans(txt, troles, tokenizer_func):
-    c = 0
-    c2 = 0
-    cr = -1
-    cs = -1
-    prev = "O"
-
-    roles = []
-    ttxt = tokenizer_func(txt)
-
-    if len(ttxt) != len(troles):
-        ttxt = ttxt[: len(troles)]
-
-    for j, tok in enumerate(ttxt):
-        if c >= len(txt):
-            break
-
-        while c < len(txt) and txt[c].isspace():
-            c += 1
-
-        if tok[:2] == "##" or tok == "[UNK]":
-            c += len(tok) - 2 if tok[:2] == "##" else 1
-        else:
-            if troles[j].startswith("B-"):
-                if cs >= cr:
-                    cr = c
-                    if cs >= 0:
-                        roles.append({"start": cs, "end": c2, "label": prev})
-                cs = c
-                prev = troles[j][2:]
-            else:
-                if troles[j] == "O":
-                    if cs >= cr:
-                        cr = c
-                        if cs >= 0:
-                            roles.append({"start": cs, "end": c2, "label": prev})
-            c += len(tok)
-        c2 = c
-
-    if cs >= cr:
-        if cs >= 0:
-            roles.append({"start": cs, "end": c2, "label": prev})
-
-    return roles
-
-
-def span2bio(txt, labels):
-    roles = sorted(labels, key=lambda x: x["label"])
-    roles_left = [r["start"] for r in roles]
-
-    ttxt = re.findall(r"[{}]|\w+".format(string.punctuation), txt)
-
-    c = 0
-    cr = -1
-    prev = "O"
-    troles = []
-    for tok in ttxt:
-        if c >= len(txt):
-            break
-
-        while txt[c] == " ":
-            c += 1
-
-        else:
-            if c in roles_left:  # Start of a new role
-                ind = roles_left.index(c)
-                cr = roles[ind]["end"]
-                prev = "I-" + roles[ind]["label"]
-                troles.append("B-" + roles[ind]["label"])
-            else:
-                if c < cr:  # Assign previous role
-                    troles.append(prev)
-                else:  # Assign 'O'
-                    troles.append("O")
-
-            c += len(tok)
-
-    if len(ttxt) != len(troles):
-        troles += ["O"] * (len(ttxt) - len(troles))
-
-    assert len(ttxt) == len(troles)
-    return ttxt, troles
+from ner_helpers import span2bio
 
 
 def load_json(file_path):
@@ -176,9 +38,18 @@ def evaluate_bail(gold_data, pred_data):
 
     f1 = f1_score(gold_labels, pred_labels, average="macro")
     print("Macro-F1 on HLDC-all-districts test set:", f1)
+    return f1
 
-    return f"{f1:.2f}"
-
+def get_BLEU_score(ref_text_all, machine_text_all):
+    sc_all = []
+    for i in range(len(ref_text_all)):
+        ref_text = ref_text_all[i]
+        machine_text = machine_text_all[i]
+        tok_ref_text = nltk.word_tokenize(ref_text)
+        tok_machine_text = nltk.word_tokenize(machine_text)
+        sc = nltk.translate.bleu_score.sentence_bleu([tok_ref_text], tok_machine_text, weights = (0.5,0.5))
+        sc_all.append(sc)
+    return sum(sc_all)/len(sc_all)
 
 def evaluate_cjpe(gold_data, pred_data):
     # Evaluate prediction
@@ -191,48 +62,76 @@ def evaluate_cjpe(gold_data, pred_data):
     f1 = f1_score(gold_labels, pred_labels, average="macro")
     prediction_result = {"cjpe-eval": f1}
 
-    # Evaluate explanation
-    rouge = evaluate.load("rouge")
-    bleu = evaluate.load("bleu")
-
-    gold_explanations = [exp["expert_1"] for exp in gold_data["explanation"].values()]
-    pred_explanations = [exp["expert_1"] for exp in pred_data["explanation"].values()]
-
-    rouge_scores = rouge.compute(
-        predictions=pred_explanations, references=gold_explanations
-    )
-    bleu_score = bleu.compute(
-        predictions=pred_explanations, references=gold_explanations
-    )
+    R = []
+    B = []
+    rl_evaluator = rouge.Rouge(metrics=['rouge-l'], max_n=2, limit_length=False, apply_avg=True)
+    for x in range(1, 6):
+        gold_explanations = []
+        pred_explanations = []
+        for k,v in gold_data['explanation'].items():
+            gold_explanations.append(v[f'expert_{x}'])
+            pred_explanations.append(pred_data['explanation'][k])
+        rougex = rl_evaluator.get_scores(pred_explanations, gold_explanations)['rouge-l']['f']
+        bleux = get_BLEU_score(gold_explanations, pred_explanations)
+        R.append(rougex)
+        B.append(bleux)
+    
+    rouge_score = sum(R)/len(R)
+    bleu_score = sum(B)/len(B)
 
     explanation_result = {
         "cjpe-exp-eval": {
-            "rouge": [rouge_scores],
-            "bleu": [bleu_score],
+            "rouge": rouge_score,
+            "bleu": bleu_score,
         }
     }
-
+    print("Macro-F1 on ILDC test:", prediction_result)
+    print("Explanability for ILDC Expert:", explanation_result)
     return {**prediction_result, **explanation_result}
 
+def span2bio(txt, roles):
+    roles = sorted(roles, key = lambda x:x['start'])        
+    roles_left = [r['start'] for r in roles]    
+    
+    ttxt = re.findall(r'[{}]|\w+'.format(string.punctuation), txt)
+                            
+    c = 0
+    cr = -1
+    prev = 'O'
+    troles = []
+    for tok in ttxt:
+        if c >= len(txt):
+            break
+        
+        while txt[c] == ' ':
+            c += 1
+        
+        else:
+            if c in roles_left: # Start of a new role
+                ind = roles_left.index(c)
+                cr = roles[ind]['end']
+                prev = 'I-' + roles[ind]['label']
+                troles.append('B-' + roles[ind]['label'])
+            else:
+                if c < cr: # Assign previous role
+                    troles.append(prev)
+                else: # Assign 'O'
+                    troles.append('O')
+            
+            c += len(tok)
+        
+    if len(ttxt) != len(troles):
+        troles += ['O'] * (len(ttxt) - len(troles))
+        
+    assert len(ttxt) == len(troles)
+    return ttxt, troles
 
 def evaluate_lner(gold_data, pred_data, text_data):
-    labels = [
-        "APP",
-        "RESP",
-        "A.COUNSEL",
-        "R.COUNSEL",
-        "JUDGE",
-        "WIT",
-        "AUTH",
-        "COURT",
-        "STAT",
-        "PREC",
-        "DATE",
-        "CASENO",
-    ]
+    with open("ner_labels.txt") as f:
+        labels = f.read().strip().split("\n")
 
     results_per_fold = {}
-    for fold in range(1, 4):
+    for fold in range(1, len(gold_data) + 1):
         gold = gold_data[f"fold_{fold}"]
         pred = pred_data[f"fold_{fold}"]
         text = text_data[f"fold_{fold}"]
@@ -251,6 +150,7 @@ def evaluate_lner(gold_data, pred_data, text_data):
             pred_labels.append(pred_bio)
 
         evaluator = Evaluator(gold_labels, pred_labels, tags=labels, loader="list")
+
         results, results_per_tag, _, _ = evaluator.evaluate()
 
         f1_scores = [results_per_tag[l]["strict"]["f1"] for l in results_per_tag]
@@ -258,22 +158,34 @@ def evaluate_lner(gold_data, pred_data, text_data):
         print(f"Strict Macro-F1 on Fold {fold}:", avg_f1)
         results_per_fold[f"fold_{fold}"] = avg_f1
 
-    return {"strict mF1": f"{np.mean(list(results_per_fold.values()))}:.2f"}
+    print("Strict macro-F1 on L-NER Dataset:", results_per_fold)
+    return results_per_fold
 
 
 def evaluate_rr(gold_data, pred_data):
     all_gold_labels = []
     all_pred_labels = []
+    with open("rr_label_vocab.json") as f:
+        label_vocab = json.load(f)
+
 
     for id, gold_labels in gold_data.items():
         pred_labels = pred_data.get(id, ["None"] * len(gold_labels))
-        all_gold_labels.extend(gold_labels)
-        all_pred_labels.extend(pred_labels)
-
-    mf1 = f1_score(all_gold_labels, all_pred_labels, average="macro")
-    print(f"Macro-F1 on combined test set:", mf1)
-
-    return {"mF1": f"{mf1:.2f}"}
+        for i in range(len(gold_labels)):
+            g = gold_labels[i]
+            p = pred_labels[i]
+            if g not in label_vocab: continue
+            for pp in p.split():
+                if pp in label_vocab:
+                    p = pp
+                    break
+            if p not in label_vocab: continue
+            all_gold_labels.append([label_vocab[g]])
+            all_pred_labels.append([label_vocab[p]])
+
+    f1 = f1_score(all_gold_labels, all_pred_labels, average="macro")
+    print(f"Macro-F1 on combined test set:", f1)
+    return f1
 
 
 def evaluate_lsi(gold_data, pred_data):
@@ -324,7 +236,7 @@ def evaluate_pcr(gold_data, pred_data):
 
         print(f"Micro-F1@{k} on IL-PCR test set:", f1)
 
-    return np.mean(f1_scores)
+    return f1_scores
 
 
 def evaluate_summ(gold_data, pred_data):
@@ -339,11 +251,12 @@ def evaluate_summ(gold_data, pred_data):
             gold_summaries.append(gold_summary)
             pred_summaries.append(pred_summary)
 
-    rouge = evaluate.load("rouge")
-    rouge_scores = rouge.compute(predictions=pred_summaries, references=gold_summaries)
-    print("Rouge-L:", rouge_scores)
+    rl_evaluator = rouge.Rouge(metrics=['rouge-n','rouge-l'], max_n=2, limit_length=False, apply_avg=True)
+    rl_scores = rl_evaluator.get_scores(pred_summaries, gold_summaries)
 
-    return {"ROUGE-L": rouge_scores, "BERTSCORE": "-"}
+    _, _, bs = bert_score.score(pred_summaries, gold_summaries, lang="en", verbose=True, device='cuda')
+    print("Rouge:", {k:v['f'] for k,v in rl_scores.items()}, "BERTSCORE:", bs.mean().item())
+    return {'ROUGE': rl_scores['rouge-l']['f'], 'BERTSCORE': bs.mean().item()}
 
 
 def evaluate_lmt(gold_data, pred_data):
@@ -423,8 +336,8 @@ def create_output_json(evaluation_results):
 def main():
     # gold_data = load_json("IL_TUR_eval_gold.json")
     # pred_data = load_json("IL_TUR_eval_submission2.json")
-    gold_data = load_json("submissions/baseline/IL_TUR_eval_gold_small.json")
-    pred_data = load_json("submissions/baseline/IL_TUR_eval_submission_small.json")
+    gold_data = load_json("submissions/baseline/IL_TUR_eval_gold.json")
+    pred_data = load_json("submissions/baseline/IL_TUR_eval_submission_dummy.json")
     pred_data = gold_data
     evaluation_results = {}