awrawrawr

app.py

@@ -3,4 +3,8 @@ from evaluate.utils import launch_gradio_widget
 
 
 module = evaluate.load("brian920128/doc_retrieve_metrics")
-launch_gradio_widget(module)
+
+
+result = module.compute(references=[[["e", ""], ["", ""]]], predictions=[[["", ""]]])
+print(result)
+#launch_gradio_widget(module)

doc_retrieve_metrics.py

@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """TODO: Add a description here."""
+from typing import List
 
 import evaluate
 import datasets
 
-
 # TODO: Add BibTeX citation
 _CITATION = """\
 @InProceedings{huggingface:module,
@@ -31,7 +31,6 @@ _DESCRIPTION = """\
 This new module is designed to solve this great ML task and is crafted with a lot of care.
 """
 
-
 # TODO: Add description of the arguments of the module here
 _KWARGS_DESCRIPTION = """
 Calculates how good are predictions given some references, using certain scores
@@ -53,8 +52,49 @@ Examples:
     {'accuracy': 1.0}
 """
 
-# TODO: Define external resources urls if needed
-BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
+
+def calculate_precision(
+        predictions: List[List[str]],
+        reference: List[List[str]]
+) -> float:
+    precision = 0
+    count = 0
+
+    for i, d in enumerate(reference):
+        if len(d) == 0:
+            continue
+
+        predicted_pages = predictions[i]
+        hits = predicted_pages.intersection(d)
+        if len(predicted_pages) != 0:
+            precision += len(hits) / len(predicted_pages)
+
+        count += 1
+
+    return precision / count
+
+
+def calculate_recall(
+        predictions: List[List[str]],
+        reference: List[List[str]]
+) -> float:
+    recall = 0
+    count = 0
+
+    for i, d in enumerate(reference):
+        if len(d) == 0:
+            continue
+
+        predicted_pages = predictions[i]
+        hits = predicted_pages.intersection(d)
+        recall += len(hits) / len(d)
+
+        count += 1
+
+    return recall / count
+
+
+beta = 0.7
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
@@ -87,9 +127,11 @@ class DocRetrieveMetrics(evaluate.Metric):
         pass
 
     def _compute(self, predictions, references):
-        """Returns the scores"""
-        # TODO: Compute the different scores of the module
-        accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
+        recall = calculate_recall(predictions, references)
+        precision = calculate_precision(predictions, references)
+        f_score = (1 + beta) * precision * recall / (beta * beta)(precision + recall)
         return {
-            "accuracy": accuracy,
-        }
+            "f1": float(
+                f_score
+            )
+        }