First version

README.md

@@ -1,7 +1,5 @@
 ---
 title: Metrica Tesi
-datasets:
-- placeholder 
 tags:
 - evaluate
 - metric

metrica_tesi.py

@@ -15,6 +15,7 @@
 
 import evaluate
 import datasets
+from itertools import repeat
 
 
 # TODO: Add BibTeX citation
@@ -36,13 +37,15 @@ This new module is designed to solve this great ML task and is crafted with a lo
 _KWARGS_DESCRIPTION = """
 Calculates how good are predictions given some references, using certain scores
 Args:
-    predictions: list of predictions to score. Each predictions
+    predictions: list of predictions to score. Each prediction
         should be a string with tokens separated by spaces.
+        Special tokens must be included.
     references: list of reference for each prediction. Each
-        reference should be a string with tokens separated by spaces.
+        reference should be a string with tokens separated by spaces.  
+        Special tokens must be included.
+    n: number of last tokens to be considered for the calculation.
 Returns:
-    accuracy: description of the first score,
-    another_score: description of the second score,
+    score: accuracy score calculated on the last n action tokens of every pair prediction-reference.
 Examples:
     Examples should be written in doctest format, and should illustrate how
     to use the function.
@@ -57,6 +60,10 @@ Examples:
 BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
 
 
+def get_last_n_tokens(string_of_tokens, n):
+    return string_of_tokens.split(" ")[-n:]
+
+
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class MetricaTesi(evaluate.Metric):
     """TODO: Short description of my evaluation module."""
@@ -70,15 +77,17 @@ class MetricaTesi(evaluate.Metric):
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,
             # This defines the format of each prediction and reference
-            features=datasets.Features({
-                'predictions': datasets.Value('int64'),
-                'references': datasets.Value('int64'),
-            }),
+            features=datasets.Features(
+                {
+                    "predictions": datasets.Value("int64"),
+                    "references": datasets.Value("int64"),
+                }
+            ),
             # Homepage of the module for documentation
             homepage="http://module.homepage",
             # Additional links to the codebase or references
             codebase_urls=["http://github.com/path/to/codebase/of/new_module"],
-            reference_urls=["http://path.to.reference.url/new_module"]
+            reference_urls=["http://path.to.reference.url/new_module"],
         )
 
     def _download_and_prepare(self, dl_manager):
@@ -86,10 +95,14 @@ class MetricaTesi(evaluate.Metric):
         # TODO: Download external resources if needed
         pass
 
-    def _compute(self, predictions, references):
+    def _compute(self, predictions, references, n):
         """Returns the scores"""
         # TODO: Compute the different scores of the module
-        accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
+        score = 0
+        for pair in zip(map(get_last_n_tokens, predictions, repeat(n)), map(get_last_n_tokens, references, repeat(n))):
+            if len(pair[0]) == 2:
+                score += sum(pred_token == ref_token for pred_token, ref_token in zip(*pair)) / 2
+        score /= len(predictions)
         return {
-            "accuracy": accuracy,
-        }
+            "score": score,
+        }

tests.py

@@ -1,17 +1,17 @@
 test_cases = [
     {
-        "predictions": [0, 0],
-        "references": [1, 1],
+        "predictions": ["blu red yellow", "green orange violet"],
+        "references": ["pizza spaghetti mandolino", "piplup prinplup empoleon"],
         "result": {"metric_score": 0}
     },
     {
-        "predictions": [1, 1],
-        "references": [1, 1],
+        "predictions": ["a random string", "another string maybe"],
+        "references": ["random random string", "string string maybe"],
         "result": {"metric_score": 1}
     },
     {
-        "predictions": [1, 0],
-        "references": [1, 1],
+        "predictions": ["conda search evaluate", "conda search transformers"],
+        "references": ["conda search evaluate", "conda install pytorch"],
         "result": {"metric_score": 0.5}
     }
 ]