test

src/about.py

@@ -22,20 +22,70 @@ class Tasks(Enum):
 class AssetTasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
     task0 = Task("acc_electric_motor", "acc_electric_motor", "acc_electric_motor")
-    # task1 = Task("acc_steam_turbine", "acc_steam_turbine", "acc_steam_turbine")
-    # task2 = Task("acc_aero_gas_turbine", "acc_aero_gas_turbine", "acc_aero_gas_turbine")
-    # task3 = Task("acc_industrial_gas_turbine", "acc_industrial_gas_turbine", "acc_industrial_gas_turbine")
-    # task4 = Task("acc_pump", "acc_pump", "acc_pump")
-    # task5 = Task("acc_compressor", "acc_compressor", "acc_compressor")
-    # task6 = Task("acc_reciprocating_internal_combustion_engine", "acc_reciprocating_internal_combustion_engine", "acc_reciprocating_internal_combustion_engine")
-    # task7 = Task("acc_electric_generator", "acc_electric_generator", "acc_electric_generator")
-    # task8 = Task("acc_fan", "acc_fan", "acc_fan")
-    # task9 = Task("acc_power_transformer", "acc_power_transformer", "acc_power_transformer")
-
+    task1 = Task("acc_steam_turbine", "acc_steam_turbine", "acc_steam_turbine")
+    task2 = Task("acc_aero_gas_turbine", "acc_aero_gas_turbine", "acc_aero_gas_turbine")
+    task3 = Task("acc_industrial_gas_turbine", "acc_industrial_gas_turbine", "acc_industrial_gas_turbine")
+    task4 = Task("acc_pump", "acc_pump", "acc_pump")
+    task5 = Task("acc_compressor", "acc_compressor", "acc_compressor")
+    task6 = Task("acc_reciprocating_internal_combustion_engine", "acc_reciprocating_internal_combustion_engine", "acc_reciprocating_internal_combustion_engine")
+    task7 = Task("acc_electric_generator", "acc_electric_generator", "acc_electric_generator")
+    task8 = Task("acc_fan", "acc_fan", "acc_fan")
+    task9 = Task("acc_power_transformer", "acc_power_transformer", "acc_power_transformer")
+
+
+# {
+#   "acc_overall": {
+#     "acc": 0.38732658417697785
+#   },
+#   "acc_sel": {
+#     "acc_sel": 0.40638297872340423
+#   },
+#   "acc_el": {
+#     "acc_el": 0.4954128440366973
+#   },
+#   "acc_perturb": {
+#     "perturb_score": 0.2819647544056993
+#   },
+#   "score_consistency": {
+#     "consist_score": 0.16422947131608548
+#   },
+#   "uncertainty": {
+#     "uncertainty_score": 0
+#   },
+#   "acc_electric_motor": {
+#     "acc_electric_motor": 0.41025641025641024
+#   },
+#   "acc_steam_turbine": {
+#     "acc_steam_turbine": 0.30409356725146197
+#   },
+#   "acc_aero_gas_turbine": {
+#     "acc_aero_gas_turbine": 0.3541666666666667
+#   },
+#   "acc_industrial_gas_turbine": {
+#     "acc_industrial_gas_turbine": 0.45
+#   },
+#   "acc_pump": {
+#     "acc_pump": 0.39473684210526316
+#   },
+#   "acc_compressor": {
+#     "acc_compressor": 0.35
+#   },
+#   "acc_reciprocating_internal_combustion_engine": {
+#     "acc_reciprocating_internal_combustion_engine": 0.47619047619047616
+#   },
+#   "acc_electric_generator": {
+#     "acc_electric_generator": 0.42735042735042733
+#   },
+#   "acc_fan": {
+#     "acc_fan": 0.445
+#   },
+#   "acc_power_transformer": {
+#     "acc_power_transformer": 0.3161764705882353
+#   }
+# }
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
-# "results": {"acc_overall": {"acc": 0.22797150356205476}, "acc_sel": {"acc_sel": 0.16170212765957448}, "acc_el": {"acc_el": 0.3669724770642202}, "acc_perturb": {"perturb_score": 0.161604799400075}, "score_consistency": {"consist_score": 0.048743907011623545}, "uncertainty": {"uncertainty_score": 0}}, "acc_electric motor": 0.2863247863247863, "acc_steam turbine": 0.19298245614035087, "acc_aero gas turbine": 0.1875, "acc_industrial gas turbine": 0.19166666666666668, "acc_pump": 0.23684210526315788, "acc_compressor": 0.20454545454545456, "acc_reciprocating internal combustion engine": 0.32142857142857145, "acc_electric generator": 0.2222222222222222, "acc_fan": 0.27, "acc_power transformer": 0.19117647058823528}
 
 
 # Your leaderboard name

src/leaderboard/read_evals.py

@@ -107,7 +107,7 @@ class EvalResult:
         except Exception:
             print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
 
-    def to_dict(self):
+    def to_dict(self, task_class):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
@@ -126,7 +126,7 @@ class EvalResult:
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
-        for task in Tasks:
+        for task in task_class:
             data_dict[task.value.col_name] = self.results[task.value.benchmark]
 
         return data_dict
@@ -186,7 +186,7 @@ def get_raw_eval_results(results_path: str, requests_path: str, task_class) -> l
     results = []
     for v in eval_results.values():
         try:
-            v.to_dict() # we test if the dict version is complete
+            v.to_dict(task_class) # we test if the dict version is complete
             results.append(v)
         except KeyError:  # not all eval values present
             print(f'error key {str(v)}')