add metric

ecqa.py

@@ -15,7 +15,9 @@
 
 import evaluate
 import datasets
-
+import re
+import string
+from collections import Counter
 
 # TODO: Add BibTeX citation
 _CITATION = """\
@@ -56,6 +58,34 @@ Examples:
 # TODO: Define external resources urls if needed
 BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
 
+def remove_(text: str)-> str:
+    ''' 불필요한 기호 제거 '''
+    text = re.sub("'", " ", text)
+    text = re.sub('"', " ", text)
+    text = re.sub('《', " ", text)
+    text = re.sub('》', " ", text)
+    text = re.sub('<', " ", text)
+    text = re.sub('>', " ", text)
+    text = re.sub('〈', " ", text)
+    text = re.sub('〉', " ", text)
+    text = re.sub("\(", " ", text)
+    text = re.sub("\)", " ", text)
+    text = re.sub("‘", " ", text)
+    text = re.sub("’", " ", text)
+    return text
+
+def white_space_fix(text: str)-> str:
+    '''연속된 공백일 경우 하나의 공백으로 대체'''
+    return ' '.join(text.split())
+
+def remove_punc(text: str)-> str:
+    '''구두점 제거'''
+    exclude = set(string.punctuation)
+    return ''.join(ch for ch in text if ch not in exclude)
+
+def lower(text: str)-> str:
+    '''소문자 전환'''
+    return text.lower()
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class ecqa(evaluate.Metric):
@@ -85,11 +115,59 @@ class ecqa(evaluate.Metric):
         """Optional: download external resources useful to compute the scores"""
         # TODO: Download external resources if needed
         pass
+    
+    def __normalize(self, text: str):
+        text = remove_(text)
+        text = lower(text)
+        text = remove_punc(text)
+        return white_space_fix(text)
+
+    def __compute_f1(self, prediction: str, reference: str)-> tuple[float, float, float]:
+        predicted_tokens = prediction.split()
+        referenced_tokens = reference.split()
+
+        predictied_chars = []
+        for token in predicted_tokens:
+            predictied_chars += [char for char in token]
 
-    def _compute(self, predictions, references):
+        referenced_chars = []
+        for token in referenced_tokens:
+            referenced_chars += [char for char in token]
+
+        true_positive = Counter(predictied_chars) & Counter(referenced_chars)
+        n_true_positive = sum(true_positive.values())
+        precision = 1.0 * n_true_positive / len(predictied_chars)
+        recall = 1.0 * n_true_positive / len(referenced_chars)
+        f1 = (2 * precision * recall) / (precision + recall)
+
+        return f1, recall, precision
+    def _compute(self, predictions: list[str], references: list[str]):
         """Returns the scores"""
         # TODO: Compute the different scores of the module
-        accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
+        assert isinstance(predictions, list)
+        assert isinstance(references, list)
+        assert len(predictions) == len(references)
+        f1_acc = precision_acc = recall_acc = total = 0
+        for prediction, reference in zip(predictions, references):
+            total += 1
+            f1_computed, precision_computed, recall_computed = self.__compute_f1(prediction, reference)
+
+            f1_acc += f1_computed
+            precision_acc += precision_computed
+            recall_acc += recall_computed
+
+        f1, precision, recall = [
+            # average
+            100.0 * computed / total
+
+            for computed in [
+                f1_acc,
+                precision_acc,
+                recall_acc
+            ]
+        ]
         return {
-            "accuracy": accuracy,
+            "f1": f1,
+            "precision": precision,
+            "recall": recall
         }