dfs

src/about.py

@@ -17,7 +17,8 @@ class Tasks(Enum):
     task2 = Task("VCR", "acc", "VCR")
     task3 = Task("Culture", "acc", "Culture")
     task4 = Task("Trick", "acc", "Trick")
-
+    
+class N_Tasks(Enum):
     task0_f1 = Task("Count", "f1", "Count")
     task1_f1 = Task("Order", "f1", "Order")
     task2_f1 = Task("VCR", "f1", "VCR")

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks
+from src.about import Tasks, N_Tasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]

src/leaderboard/read_evals.py

@@ -8,7 +8,8 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, Precision, WeightType
+from src.about import Tasks, N_Tasks
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -114,9 +115,9 @@ class EvalResult:
         except Exception:
             print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
 
-    def to_dict(self):
+    def to_dict(self, tasks):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        average = sum([v for v in self.results.values() if v is not None]) / len(tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.architecture.name: self.architecture,
@@ -127,7 +128,7 @@ class EvalResult:
             AutoEvalColumn.dataset_version.name: self.dataset_version,
         }
 
-        for task in Tasks:
+        for task in tasks:
             data_dict[task.value.col_name] = self.results[task.value.benchmark]
 
         return data_dict
@@ -187,10 +188,12 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         else:
             eval_results[eval_name] = eval_result
 
+    version = results_path.split("/")[-1]
+    tasks = N_Tasks if "n_" in version else Tasks
     results = []
     for v in eval_results.values():
         try:
-            v.to_dict() # we test if the dict version is complete
+            v.to_dict(tasks) # we test if the dict version is complete
             results.append(v)
         except KeyError:  # not all eval values present
             continue