Update src/leaderboard/read_evals.py

src/leaderboard/read_evals.py

@@ -45,9 +45,6 @@ class EvalResult:
         config = data.get("config")
         metainfo = config.get("metainfo", {})
         model_config = config.get("model", {})
-        print(f"Config: {config}")
-        print(f"Metainfo: {metainfo}")
-        print(f"Model config: {model_config}")
 
         # Get model type from metainfo
         # model_type_str = metainfo.get("model_type", "") 
@@ -60,7 +57,6 @@ class EvalResult:
         # Precision
         # precision = Precision.from_str(config.get("dtype"))
         precision = model_config.get("dtype", "Unknown")
-        print(f"Precision: {precision}")
 
         # Add Special Tokens
         add_special_tokens = str(config.get("pipeline_kwargs",{"add_special_tokens":"Unknown"}).get("add_special_tokens"))
@@ -170,6 +166,27 @@ class EvalResult:
         return data_dict
 
 
+# def get_request_file_for_model(requests_path, model_name, precision):
+#     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+#     request_files = os.path.join(
+#         requests_path,
+#         f"{model_name}_eval_request_*.json",
+#     )
+#     request_files = glob.glob(request_files)
+
+#     # Select correct request file (precision)
+#     request_file = ""
+#     request_files = sorted(request_files, reverse=True)
+#     for tmp_request_file in request_files:
+#         with open(tmp_request_file, "r") as f:
+#             req_content = json.load(f)
+#             if (
+#                 req_content["status"] in ["FINISHED"]
+#                 and req_content["precision"] == precision.split(".")[-1]
+#             ):
+#                 request_file = tmp_request_file
+#     return request_file
+
 def get_request_file_for_model(requests_path, model_name, precision):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
     request_files = os.path.join(
@@ -178,20 +195,35 @@ def get_request_file_for_model(requests_path, model_name, precision):
     )
     request_files = glob.glob(request_files)
 
-    # Select correct request file (precision)
+    # Precisionの候補をリクエストファイルから動的に取得
+    precision_candidates = []
+    for request_file in request_files:
+        # ファイル名から precision を抽出する (例: _bfloat16_ や _float16_ を検出)
+        if "_bfloat16_" in request_file:
+            precision_candidates.append("bfloat16")
+        elif "_float16_" in request_file:
+            precision_candidates.append("float16")
+    
+    # precisionがUnknownの場合、最初の候補を使用する
+    if precision == "Unknown" and precision_candidates:
+        precision = precision_candidates[0]  # 最初の精度を使用（他の候補も考慮したければ処理を変更）
+    
+    print(f"Final precision used for request file search: {precision}")
+    
+    # precisionに基づいてファイルを検索
     request_file = ""
-    request_files = sorted(request_files, reverse=True)
     for tmp_request_file in request_files:
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
             if (
                 req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
+                and req_content["precision"] == precision
             ):
                 request_file = tmp_request_file
     return request_file
 
 
+
 def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []