Update src/leaderboard/read_evals.py

src/leaderboard/read_evals.py

@@ -174,27 +174,6 @@ class EvalResult:
         return data_dict
 
 
-# def get_request_file_for_model(requests_path, model_name, precision):
-#     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-#     request_files = os.path.join(
-#         requests_path,
-#         f"{model_name}_eval_request_*.json",
-#     )
-#     request_files = glob.glob(request_files)
-
-#     # Select correct request file (precision)
-#     request_file = ""
-#     request_files = sorted(request_files, reverse=True)
-#     for tmp_request_file in request_files:
-#         with open(tmp_request_file, "r") as f:
-#             req_content = json.load(f)
-#             if (
-#                 req_content["status"] in ["FINISHED"]
-#                 and req_content["precision"] == precision.split(".")[-1]
-#             ):
-#                 request_file = tmp_request_file
-#     return request_file
-
 def get_request_file_for_model(requests_path, model_name, precision):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
     request_files = os.path.join(
@@ -203,33 +182,54 @@ def get_request_file_for_model(requests_path, model_name, precision):
     )
     request_files = glob.glob(request_files)
 
-    # Precisionの候補をリクエストファイルから動的に取得
-    precision_candidates = []
-    for request_file in request_files:
-        # ファイル名から precision を抽出する (例: _bfloat16_ や _float16_ を検出)
-        if "_bfloat16_" in request_file:
-            precision_candidates.append("bfloat16")
-        elif "_float16_" in request_file:
-            precision_candidates.append("float16")
-    
-    # precisionがUnknownの場合、最初の候補を使用する
-    if precision == "Unknown" and precision_candidates:
-        precision = precision_candidates[0]  # 最初の精度を使用（他の候補も考慮したければ処理を変更）
-    
-    print(f"Final precision used for request file search: {precision}")
-    
-    # precisionに基づいてファイルを検索
+    # Select correct request file (precision)
     request_file = ""
+    request_files = sorted(request_files, reverse=True)
     for tmp_request_file in request_files:
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
             if (
                 req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision
+                and req_content["precision"] == precision.split(".")[-1]
             ):
                 request_file = tmp_request_file
     return request_file
 
+# def get_request_file_for_model(requests_path, model_name, precision):
+#     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+#     request_files = os.path.join(
+#         requests_path,
+#         f"{model_name}_eval_request_*.json",
+#     )
+#     request_files = glob.glob(request_files)
+
+#     # Precisionの候補をリクエストファイルから動的に取得
+#     precision_candidates = []
+#     for request_file in request_files:
+#         # ファイル名から precision を抽出する (例: _bfloat16_ や _float16_ を検出)
+#         if "_bfloat16_" in request_file:
+#             precision_candidates.append("bfloat16")
+#         elif "_float16_" in request_file:
+#             precision_candidates.append("float16")
+    
+#     # precisionがUnknownの場合、最初の候補を使用する
+#     if precision == "Unknown" and precision_candidates:
+#         precision = precision_candidates[0]  # 最初の精度を使用（他の候補も考慮したければ処理を変更）
+    
+#     print(f"Final precision used for request file search: {precision}")
+    
+#     # precisionに基づいてファイルを検索
+#     request_file = ""
+#     for tmp_request_file in request_files:
+#         with open(tmp_request_file, "r") as f:
+#             req_content = json.load(f)
+#             if (
+#                 req_content["status"] in ["FINISHED"]
+#                 and req_content["precision"] == precision
+#             ):
+#                 request_file = tmp_request_file
+#     return request_file
+
 
 
 def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]: