Make values clickable

src/about.py

@@ -6,6 +6,7 @@ class Task:
     benchmark: str
     metric: str
     col_name: str
+    type: str
 
 
 # Select your tasks here
@@ -14,22 +15,22 @@ class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
 
     # base
-    task0 = Task("arc_easy", "accuracy", "ARC-Easy")
-    task1 = Task("arc_challenge", "accuracy", "ARC-Challenge")
-    task2 = Task("drop", "mean", "DROP")
-    task3 = Task("winogrande", "accuracy", "WinoGrande")
-    task4 = Task("gsm8k", "accuracy", "GSM8K")
-    task5 = Task("hellaswag", "accuracy", "HellaSwag")
-    task6 = Task("humaneval", "mean", "HumanEval")
-    task7 = Task("ifeval", "final_acc", "IFEval")
-    task8 = Task("math", "accuracy", "MATH")
-    task9 = Task("mmlu", "accuracy", "MMLU")
-    task10 = Task("mmlu_pro", "accuracy", "MMLU-Pro")
-    task11 = Task("gpqa_diamond", "accuracy", "GPQA-Diamond")
+    task0 = Task("arc_easy", "accuracy", "ARC-Easy", "base")
+    task1 = Task("arc_challenge", "accuracy", "ARC-Challenge", "base")
+    task2 = Task("drop", "mean", "DROP", "base")
+    task3 = Task("winogrande", "accuracy", "WinoGrande", "base")
+    task4 = Task("gsm8k", "accuracy", "GSM8K", "base")
+    task5 = Task("hellaswag", "accuracy", "HellaSwag", "base")
+    task6 = Task("humaneval", "mean", "HumanEval", "base")
+    task7 = Task("ifeval", "final_acc", "IFEval", "base")
+    task8 = Task("math", "accuracy", "MATH", "base")
+    task9 = Task("mmlu", "accuracy", "MMLU", "base")
+    task10 = Task("mmlu_pro", "accuracy", "MMLU-Pro", "base")
+    task11 = Task("gpqa_diamond", "accuracy", "GPQA-Diamond", "base")
 
     # agentic
-    task12 = Task("gaia", "mean", "GAIA")
-    task13 = Task("gdm_intercode_ctf", "accuracy", "GDM-InterCode-CTF")
+    task12 = Task("gaia", "mean", "GAIA", "agentic")
+    task13 = Task("gdm_intercode_ctf", "accuracy", "GDM-InterCode-CTF", "agentic")
 
 
 NUM_FEWSHOT = 0 # Change with your few shot

src/display/utils.py

@@ -28,7 +28,7 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "markdown", True)])
 # # Model information
 # auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 # auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])

src/inspect_log_file_names.json

@@ -0,0 +1,61 @@
+{
+    "gemini-1.5-pro": {
+        "mmlu": "2024-11-04T16-56-26-05-00_mmlu_Z9KrcK7x4ZLAR5nJ9JaVUe.json",
+        "humaneval": "2024-11-04T12-43-07-05-00_humaneval_5JBjtymGtK23qwVKxqidhV.json",
+        "mmlu_pro": "2024-11-04T20-13-09-05-00_mmlu-pro_Hv2ujvKLV6H7ZwQu2q8LNw.json",
+        "math": "2024-11-04T15-48-46-05-00_math_9DAZmGEfhpa3nUcmMAwqZe.json",
+        "arc_easy": "2024-11-04T12-31-43-05-00_arc-easy_eGxYWywpLuREcaCKvHa8Uk.json",
+        "gsm8k": "2024-11-04T15-15-26-05-00_gsm8k_cTebw3ugfrVz3dyPwxtdUZ.json",
+        "gpqa_diamond": "2024-11-05T09-56-31-05-00_gpqa-diamond_FBq2bnoyGYQ3NF96xQw8iy.json",
+        "ifeval": "2024-11-04T12-43-32-05-00_ifeval_mSwZ7AwA7akj5PjZbQMjgC.json",
+        "winogrande": "2024-11-04T12-40-46-05-00_winogrande_5SmD6rx47zmZvHHkQSSfHK.json",
+        "arc_challenge": "2024-11-04T12-37-36-05-00_arc-challenge_5VVApyQD22QpJoMm53EMdU.json",
+        "drop": "2024-11-04T12-44-32-05-00_drop_9dzPKVJojSVsxmiBFnej2m.json",
+        "hellaswag": "2024-11-05T13-14-31-05-00_hellaswag_N98eeftuY2pucRtgpUYk5m.json",
+        "gaia": "2024-11-15T12-53-32-05-00_gaia_NvyGRTXFrFskJfUvuLwvVr.json",
+        "gdm_intercode_ctf": "2024-11-15T16-23-23-05-00_gdm-intercode-ctf_3JrgtTMcijTUxHVaagPRYh.json"
+    },
+    "gemini-1.5-flash": {
+        "gpqa_diamond": "2024-11-04T12-47-34-05-00_gpqa-diamond_cL5kQj8DWbRfxz79piTSdy.json",
+        "arc_challenge": "2024-11-04T12-45-59-05-00_arc-challenge_YQLMHfEXqeYgGJY86EB9bp.json",
+        "math": "2024-11-04T15-25-38-05-00_math_eaYBRMFgo8p6VUUCYxnCWj.json",
+        "drop": "2024-11-04T12-52-08-05-00_drop_5i253AQzbENgHTYN4ATemV.json",
+        "mmlu_pro": "2024-11-04T19-44-13-05-00_mmlu-pro_8GrR6wUsYNkthiZNMmLa8y.json",
+        "ifeval": "2024-11-04T12-51-30-05-00_ifeval_ZATErMbLHoyxh4kDaSqy8j.json",
+        "hellaswag": "2024-11-05T23-19-25-05-00_hellaswag_MRffohuzgVjighGb8FoqSJ.json",
+        "winogrande": "2024-11-04T12-48-29-05-00_winogrande_Hmqo6Ydz3nfCnQAdUwgrbD.json",
+        "humaneval": "2024-11-04T12-50-47-05-00_humaneval_9j4rYguKeKmxEoD9VuddwX.json",
+        "arc_easy": "2024-11-04T12-39-50-05-00_arc-easy_NwmTEw6C8VSCXzzwZCFy48.json",
+        "gsm8k": "2024-11-04T15-22-21-05-00_gsm8k_hdJs3Z6XzpR5netTcWLXJT.json",
+        "mmlu": "2024-11-04T16-26-13-05-00_mmlu_QvfQ46qJen2bvxiktHu86H.json",
+        "gdm_intercode_ctf": "2024-11-15T20-52-53-05-00_gdm-intercode-ctf_oLYr3H6bFtrcmgM6EABmNt.json"
+    },
+    "c4ai-command-r-plus": {
+        "ifeval": "2024-10-30T17-23-04-04-00_ifeval_RGucUMwdGmUnRpqyMTZTzW.json",
+        "winogrande": "2024-10-30T14-42-18-04-00_winogrande_bY8yg7aRR5dCCK7NDCZEcc.json",
+        "arc_challenge": "2024-10-29T17-30-03-04-00_arc-challenge_XB7LURXEGaxskWuLtYwdnW.json",
+        "drop": "2024-10-30T12-06-30-04-00_drop_itY9cLiYAW2BF7NTeDceNd.json",
+        "math": "2024-10-30T17-26-34-04-00_math_kohBUMpMFuMsR4jz4vUNWM.json",
+        "gpqa_diamond": "2024-10-29T22-47-45-04-00_gpqa-diamond_JKpb6ya4pec9hh7uovPPCZ.json",
+        "mmlu_pro": "2024-10-31T01-11-38-04-00_mmlu-pro_gZVAuy3zMKR23BieM5PqAX.json",
+        "humaneval": "2024-10-30T17-22-23-04-00_humaneval_5ByPqUhoofSbKgvsUQNFCX.json",
+        "gsm8k": "2024-10-30T15-03-35-04-00_gsm8k_QxbfbriJsKGQAg96JyjkoT.json",
+        "hellaswag": "2024-10-30T15-18-17-04-00_hellaswag_UYyBTR6N8VJnKRmnbCrB8N.json",
+        "mmlu": "2024-10-30T21-55-26-04-00_mmlu_JUPPLTzfe3Kme6UuorPTqg.json",
+        "arc_easy": "2024-10-29T17-10-40-04-00_arc-easy_UvprihBMLXPF8JENVLRkdx.json"
+    },
+    "Qwen2.5-72B-Instruct": {
+        "arc_challenge": "2024-10-31T13-46-34-04-00_arc-challenge_FSybKYYwpXVLQag8VwpjKe.json",
+        "mmlu_pro": "2024-11-01T20-31-04-04-00_mmlu-pro_2TfSPmsVmKatntHy2CnR7A.json",
+        "gpqa_diamond": "2024-10-31T13-48-32-04-00_gpqa-diamond_8qSySicySUyNvRRYVFBKLU.json",
+        "winogrande": "2024-10-31T14-46-29-04-00_winogrande_CX692dYh53gJ6JigT9GMpa.json",
+        "mmlu": "2024-11-01T10-08-50-04-00_mmlu_AgK27yYvmAo2LxotBH7ZL9.json",
+        "hellaswag": "2024-11-01T02-55-55-04-00_hellaswag_RSk8rGcQWg3HRrLffTNoiM.json",
+        "gsm8k": "2024-11-01T01-15-16-04-00_gsm8k_3h4W6xZjXpz9oCwtgKNYzo.json",
+        "arc_easy": "2024-10-31T13-40-08-04-00_arc-easy_3JUyzfoEHxhSBUdCU2AaVC.json",
+        "math": "2024-11-01T10-06-46-04-00_math_UUpS2R9eQc9KxBxkanT2gE.json",
+        "ifeval": "2024-10-31T14-51-45-04-00_ifeval_VGxA7gTZLZSruceM9Ci37C.json",
+        "humaneval": "2024-10-31T14-49-39-04-00_humaneval_9u7khnxivCDroJoPNRFpjs.json",
+        "drop": "2024-10-31T15-03-20-04-00_drop_DDLi98VhiV2bLzuw7fx6H4.json"
+    }
+}

src/populate.py

@@ -3,19 +3,33 @@ import os
 
 import pandas as pd
 
+from src.about import Tasks
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_inspect_log_url(model_name: str, benchmark_type: str, benchmark_name: str, log_dir: str) -> str:
+TASK_NAME_INVERSE_MAP = dict()
+for task in Tasks:
+    TASK_NAME_INVERSE_MAP[task.value.col_name] = {
+        "name": task.benchmark,
+        "type": task.type,
+    }
+
+
+def get_inspect_log_url(model_name: str, benchmark_name: str) -> str:
     """Returns the URL to the log file for a given model and benchmark"""
-    model_name = model_name.replace("/", "_")
-    log_name = f"{model_name}_{benchmark_type}.log"
-    log_path = os.path.join(log_dir, log_name)
-    if os.path.exists(log_path):
-        return log_path
-    return ""
+    with open("./inspect_log_file_names.json", "r") as f:
+        inspect_log_files = json.load(f)
+    log_file_name = inspect_log_files[model_name].get(benchmark_name, None)
+    if log_file_name is None:
+        return ""
+    else:
+        # TMP: Debugging
+        print_str = f"https://storage.googleapis.com/inspect-evals/{model_name}/index.html?log_file=logs/logs/{log_file_name}"
+        print(print_str)
+        return f"https://storage.googleapis.com/inspect-evals/{model_name}/index.html?log_file=logs/logs/{log_file_name}"
+
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
@@ -30,8 +44,13 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
 
-    # TMP: Debugging
-    print((df[AutoEvalColumn.model.name].iloc[0]).split(">")[1].split("<")[0])
+    # make values clickable and link to log files
+    for col in benchmark_cols:
+        df[col] = df[[AutoEvalColumn.model.name, col]].apply(lambda x: 
+            f"[{x[col]}]({get_inspect_log_url(
+                model_name=x[AutoEvalColumn.model.name].split(">")[1].split("<")[0],
+                benchmark_name=TASK_NAME_INVERSE_MAP[col]["name"],
+            )})", axis=1)
 
     return df