Hugging Face's logo

Spaces:
vector-institute
/
eval-leaderboard
Running

App Files Community
1
jwilles commited on
Commit
eba4aa7
·
1 Parent(s): b0b7fbb

Simplify

Browse files
Files changed (13) hide show
  1. app.py +1 -84
  2. create_log_file_map.py +0 -41
  3. data/inspect_log_file_names.json +0 -200
  4. data/populate_results.py +0 -41
  5. data/results.json.bak +0 -760
  6. refactor_eval_results.py +0 -192
  7. src/about.py +2 -43
  8. src/display/utils.py +0 -102
  9. src/envs.py +0 -25
  10. src/leaderboard/read_evals.py +0 -192
  11. src/populate.py +0 -98
  12. src/submission/check_validity.py +0 -99
  13. src/submission/submit.py +0 -119
app.py CHANGED
@@ -9,91 +9,8 @@ from src.about import (
9
  TITLE,
10
  )
11
  from src.display.css_html_js import custom_css, custom_js
12
- # from src.display.utils import (
13
- # COLS,
14
- # ST_BENCHMARK_COLS,
15
- # AGENTIC_BENCHMARK_COLS,
16
- # EVAL_COLS,
17
- # AutoEvalColumn,
18
- # fields,
19
- # )
20
- # from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
21
- # from src.populate import get_evaluation_queue_df, get_leaderboard_df, TASK_NAME_INVERSE_MAP
22
- # from src.submission.submit import add_new_eval
23
  from src.display.formatting import make_clickable_field
24
 
25
-
26
- # def restart_space():
27
- # API.restart_space(repo_id=REPO_ID)
28
-
29
- # ### Space initialisation
30
- # try:
31
- # print(EVAL_REQUESTS_PATH)
32
- # snapshot_download(
33
- # repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
34
- # )
35
- # except Exception:
36
- # restart_space()
37
- # try:
38
- # print(EVAL_RESULTS_PATH)
39
- # snapshot_download(
40
- # repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
41
- # )
42
- # except Exception:
43
- # restart_space()
44
-
45
-
46
- # ST_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, ST_BENCHMARK_COLS)
47
- # AGENTIC_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, AGENTIC_BENCHMARK_COLS)
48
-
49
- # (
50
- # finished_eval_queue_df,
51
- # running_eval_queue_df,
52
- # pending_eval_queue_df,
53
- # ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
54
-
55
- # def bold_max(s):
56
- # is_max = s == s.max() # Boolean Series: True for the max value(s)
57
- # return ['font-weight: bold' if v else '' for v in is_max]
58
-
59
- # def init_leaderboard(df, benchmark_type):
60
- # if df is None or df.empty:
61
- # raise ValueError("Leaderboard DataFrame is empty or None.")
62
-
63
- # non_task_cols = ["Model"]
64
- # if benchmark_type == "agentic":
65
- # # Include agent column
66
- # non_task_cols.append("Agent")
67
- # elif benchmark_type == "base":
68
- # # Drop agent column
69
- # dataframe = dataframe.drop(columns=["Agent"])
70
- # AutoEvalColumnSubset = [c for c in fields(AutoEvalColumn) if ((c.name in non_task_cols) or (TASK_NAME_INVERSE_MAP.get(c.name, dict()).get("type", "")==benchmark_type))]
71
-
72
- # styler = dataframe.style.apply(bold_max, subset=pd.IndexSlice[:, dataframe.columns[1:]])
73
- # df.style.set_table_styles([
74
- # {'selector': 'th', 'props': [('text-align', 'center')]},
75
- # {'selector': 'td', 'props': [('text-align', 'center')]}
76
- # ])
77
- # Define a common tooltip text
78
- # tooltip_text = "This is the common tooltip"
79
-
80
- # # Create a tooltip DataFrame with the same shape as df,
81
- # # filled with the same tooltip text for each cell.
82
- # tooltips = pd.DataFrame(tooltip_text, index=df.index, columns=df.columns)
83
-
84
- # # Apply the tooltips to the DataFrame
85
- # styled_df = df.style.set_tooltips(tooltips)
86
-
87
-
88
- # return gr.components.Dataframe(
89
- # value=df,
90
- # datatype=[c.type for c in AutoEvalColumnSubset],
91
- # column_widths=["150px" if c.name != "Model" else "250px" for c in AutoEvalColumnSubset],
92
- # wrap=False,
93
- # )
94
-
95
-
96
-
97
  def build_leaderboard(type):
98
  with open('data/results.json', 'r') as f:
99
  results = json.load(f)
@@ -147,7 +64,7 @@ def build_leaderboard(type):
147
 
148
  if type == "agentic":
149
  # Include agent column as second column after Model
150
- results_df.insert(1, 'Agent', '[Basic Agent](https://inspect.ai-safety-institute.org.uk/agents.html#sec-basic-agent)')
151
 
152
  return gr.components.Dataframe(
153
  value=results_df,
 
9
  TITLE,
10
  )
11
  from src.display.css_html_js import custom_css, custom_js
 
 
 
 
 
 
 
 
 
 
 
12
  from src.display.formatting import make_clickable_field
13
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  def build_leaderboard(type):
15
  with open('data/results.json', 'r') as f:
16
  results = json.load(f)
 
64
 
65
  if type == "agentic":
66
  # Include agent column as second column after Model
67
+ results_df.insert(1, 'Agent', make_clickable_field('Basic Agent', 'https://inspect.ai-safety-institute.org.uk/agents.html#sec-basic-agent'))
68
 
69
  return gr.components.Dataframe(
70
  value=results_df,
create_log_file_map.py DELETED
@@ -1,41 +0,0 @@
1
- import json
2
- import os
3
-
4
- from collections import defaultdict
5
-
6
- from refactor_eval_results import AGENTIC_LOG_MODEL_NAME_MAP, AGENTIC_TASKS
7
-
8
-
9
- def main():
10
-
11
- base_bm_input_path = "./base_benchmarking_logs"
12
- agentic_bm_input_path = "/fs01/projects/aieng/public/inspect_evals/agentic_benchmarking_runs"
13
-
14
- log_file_map = defaultdict()
15
-
16
- for model_name in os.listdir(base_bm_input_path):
17
- log_file_map[model_name] = defaultdict(str)
18
- if os.path.isdir(os.path.join(base_bm_input_path, model_name)):
19
- for task_log_file in os.listdir(os.path.join(base_bm_input_path, model_name)):
20
- with open(os.path.join(base_bm_input_path, model_name, task_log_file), "r") as f:
21
- result = json.load(f)
22
- task_name = result["eval"]["task"].split("/")[-1]
23
- log_file_map[model_name][task_name] = task_log_file
24
-
25
- for model_name in AGENTIC_LOG_MODEL_NAME_MAP.keys():
26
- log_file_path = os.path.join(agentic_bm_input_path, AGENTIC_LOG_MODEL_NAME_MAP[model_name])
27
- if os.path.isdir(log_file_path):
28
- for task in AGENTIC_TASKS:
29
- for task_log_file in os.listdir(os.path.join(log_file_path, task)):
30
- if task_log_file.endswith(".json"):
31
- with open(os.path.join(log_file_path, task, task_log_file), "r") as f:
32
- result = json.load(f)
33
- task_name = result["eval"]["task"].split("/")[-1]
34
- log_file_map[model_name][task_name] = task_log_file
35
-
36
- with open("./inspect_log_file_names.json", "w") as f:
37
- json.dump(log_file_map, f, indent=4)
38
-
39
-
40
- if __name__ == "__main__":
41
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
data/inspect_log_file_names.json DELETED
@@ -1,200 +0,0 @@
1
- {
2
- "gemini-1.5-pro": {
3
- "mmlu": "2024-11-04T16-56-26-05-00_mmlu_Z9KrcK7x4ZLAR5nJ9JaVUe.json",
4
- "humaneval": "2024-11-04T12-43-07-05-00_humaneval_5JBjtymGtK23qwVKxqidhV.json",
5
- "mmmu_multiple_choice": "2025-01-20T23-16-04-05-00_mmmu-multiple-choice_NLmxmHYt6CJymRVVa5UsbD.json",
6
- "mmlu_pro": "2024-11-04T20-13-09-05-00_mmlu-pro_Hv2ujvKLV6H7ZwQu2q8LNw.json",
7
- "math": "2024-11-04T15-48-46-05-00_math_9DAZmGEfhpa3nUcmMAwqZe.json",
8
- "arc_easy": "2024-11-04T12-31-43-05-00_arc-easy_eGxYWywpLuREcaCKvHa8Uk.json",
9
- "mmmu_open": "2025-01-20T23-19-25-05-00_mmmu-open_CDbtEQ7tjs5zkj4ScBbzod.json",
10
- "gsm8k": "2024-11-04T15-15-26-05-00_gsm8k_cTebw3ugfrVz3dyPwxtdUZ.json",
11
- "gpqa_diamond": "2024-11-05T09-56-31-05-00_gpqa-diamond_FBq2bnoyGYQ3NF96xQw8iy.json",
12
- "ifeval": "2024-11-04T12-43-32-05-00_ifeval_mSwZ7AwA7akj5PjZbQMjgC.json",
13
- "winogrande": "2024-11-04T12-40-46-05-00_winogrande_5SmD6rx47zmZvHHkQSSfHK.json",
14
- "arc_challenge": "2024-11-04T12-37-36-05-00_arc-challenge_5VVApyQD22QpJoMm53EMdU.json",
15
- "drop": "2024-11-04T12-44-32-05-00_drop_9dzPKVJojSVsxmiBFnej2m.json",
16
- "hellaswag": "2024-11-05T13-14-31-05-00_hellaswag_N98eeftuY2pucRtgpUYk5m.json",
17
- "gaia": "2025-01-21T15-33-29-05-00_gemini-1.5-pro_gaia_merged.json",
18
- "gdm_intercode_ctf": "2025-01-21T23-59-58+00-00_gemini-1.5-pro_gdm-intercode-ctf_merged.json",
19
- "gdm_in_house_ctf": "2025-01-22T03-42-16+00-00_gemini-1.5-pro_gdm-in-house-ctf.json",
20
- "agentharm_benign": "2025-01-21T13-18-51-08-00_agentharm-benign_gP3pQPxAuCtFLiHzt2Egt7.json",
21
- "agentharm": "2025-01-21T12-45-43-08-00_agentharm_VmD26soLwmRgWPo3hpRHBr.json",
22
- "swe_bench": "2025-01-22T03-00-08+00-00_google-gemini-1.5-pro.json"
23
- },
24
- "gemini-1.5-flash": {
25
- "gpqa_diamond": "2024-11-04T12-47-34-05-00_gpqa-diamond_cL5kQj8DWbRfxz79piTSdy.json",
26
- "arc_challenge": "2024-11-04T12-45-59-05-00_arc-challenge_YQLMHfEXqeYgGJY86EB9bp.json",
27
- "math": "2024-11-04T15-25-38-05-00_math_eaYBRMFgo8p6VUUCYxnCWj.json",
28
- "mmmu_open": "2025-01-20T23-23-50-05-00_mmmu-open_L7CnETP7d49axc7L8ChEZ4.json",
29
- "drop": "2024-11-04T12-52-08-05-00_drop_5i253AQzbENgHTYN4ATemV.json",
30
- "mmlu_pro": "2024-11-04T19-44-13-05-00_mmlu-pro_8GrR6wUsYNkthiZNMmLa8y.json",
31
- "ifeval": "2024-11-04T12-51-30-05-00_ifeval_ZATErMbLHoyxh4kDaSqy8j.json",
32
- "hellaswag": "2024-11-05T23-19-25-05-00_hellaswag_MRffohuzgVjighGb8FoqSJ.json",
33
- "winogrande": "2024-11-04T12-48-29-05-00_winogrande_Hmqo6Ydz3nfCnQAdUwgrbD.json",
34
- "humaneval": "2024-11-04T12-50-47-05-00_humaneval_9j4rYguKeKmxEoD9VuddwX.json",
35
- "arc_easy": "2024-11-04T12-39-50-05-00_arc-easy_NwmTEw6C8VSCXzzwZCFy48.json",
36
- "gsm8k": "2024-11-04T15-22-21-05-00_gsm8k_hdJs3Z6XzpR5netTcWLXJT.json",
37
- "mmlu": "2024-11-04T16-26-13-05-00_mmlu_QvfQ46qJen2bvxiktHu86H.json",
38
- "mmmu_multiple_choice": "2025-01-20T23-21-33-05-00_mmmu-multiple-choice_3huWbH3SVWx7NTGwYoKbBD.json"
39
- },
40
- "o3-mini": {
41
- "math": "2025-02-06T18-33-30-05-00_math_86Gx8n4BxhpyfaSHmRcCUm.json",
42
- "humaneval": "2025-02-06T20-58-48-05-00_humaneval_Dkod7CS9RmbbogYx9aEXtx.json",
43
- "mmlu_pro": "2025-02-06T19-49-27-05-00_mmlu-pro_jz9woKfdKt8VMzqNFsy7kY.json",
44
- "gpqa_diamond": "2025-02-06T17-57-54-05-00_gpqa-diamond_2znyMtdc7X4LJufxXeXA8Z.json",
45
- "winogrande": "2025-02-06T22-50-40-05-00_winogrande_VsTW2uU2Kj66YoNoFfRfUj.json",
46
- "gsm8k": "2025-02-06T18-23-05-05-00_gsm8k_d523pJzkcvobxamhhobCRb.json",
47
- "arc_challenge": "2025-02-06T17-53-30-05-00_arc-challenge_AYFHec7wmd4jELF2Rgzfya.json",
48
- "arc_easy": "2025-02-06T17-45-57-05-00_arc-easy_Nd8NP3K48tvwLVZb8kXDwg.json",
49
- "gaia": "2025-02-05T23-21-20+00-00_gaia_hyMq8MzMm6NgAeq3dNqZSU.json",
50
- "gdm_intercode_ctf": "2025-02-05T21-43-18+00-00_gdm-intercode-ctf_gdm29C6DuTEsX9qm9ymmrC.json",
51
- "gdm_in_house_ctf": "2025-02-05T23-59-08+00-00_gdm-in-house-ctf_2zkAX5nkJoxDnVKpJL9VgW.json",
52
- "agentharm_benign": "2025-02-03T18-49-08-08-00_agentharm-benign_Gv94YFpAXaaCJqe3Fc6yr3.json",
53
- "agentharm": "2025-02-03T18-17-03-08-00_agentharm_DmN6i5HrgXHNARjsuSewjg.json",
54
- "swe_bench": "2025-02-03T06-49-09+00-00_openai-o3-mini.json"
55
- },
56
- "DeepSeek-R1": {
57
- "mmlu_pro": "2025-02-12T11-02-35-05-00_mmlu-pro_BhD89DYN9KM3k4weSDfaQK.json",
58
- "humaneval": "2025-02-03T11-45-22-05-00_humaneval_hnkHWYqrb5HxiBt2CWzCnq.json",
59
- "math": "2025-02-11T11-38-10-05-00_math_ZYFSqsWsmP5kLRLHEMWULU.json",
60
- "gsm8k": "2025-02-02T16-28-05-05-00_gsm8k_YMw6WiZkgTBQ54z5UHtDDX.json",
61
- "arc_challenge": "2025-01-30T15-42-39-05-00_arc-challenge_CviW9ro6rKBbctkwJzQstp.json",
62
- "winogrande": "2025-02-04T00-25-12-05-00_winogrande_NPgTbtqom2QSPKxeThWrdZ.json",
63
- "arc_easy": "2025-01-30T12-48-35-05-00_arc-easy_SvRDfqsHDECQtvNU7rodZH.json",
64
- "gpqa_diamond": "2025-02-11T11-37-45-05-00_gpqa-diamond_MwnVeLwyuiEAALr3M5q3dn.json"
65
- },
66
- "o1": {
67
- "winogrande": "2025-01-20T16-46-06-05-00_winogrande_YUtAdEsForRffqe4Sm3wtR.json",
68
- "humaneval": "2025-01-17T14-59-12-05-00_humaneval_RRL8GMy9NakTxUHsDVWNng.json",
69
- "mmmu_open": "2025-01-20T22-48-09-05-00_mmmu-open_oBzxJBYbvnktbbAwhoCrYK.json",
70
- "math": "2025-01-17T15-03-22-05-00_math_6BbvHFF8hLMsVYozyNLbyQ.json",
71
- "arc_easy": "2025-01-17T11-29-26-05-00_arc-easy_DFbir4BdgQDbKd52r7tRKR.json",
72
- "arc_challenge": "2025-01-17T11-44-42-05-00_arc-challenge_PsWXaBqrgv3EcTZC55gRzJ.json",
73
- "gsm8k": "2025-01-17T12-56-38-05-00_gsm8k_iD8275qeyNTgX523pn45bF.json",
74
- "gpqa_diamond": "2025-01-17T11-53-53-05-00_gpqa-diamond_EJV7ULFSQLRoFTEqsv3t6q.json",
75
- "mmlu_pro": "2025-01-20T14-02-37-05-00_mmlu-pro_EvDzvqaahQwhv6fJovN4BT.json",
76
- "mmmu_multiple_choice": "2025-01-20T21-04-57-05-00_mmmu-multiple-choice_MctxjookaeTLCL8KpUeazT.json",
77
- "gaia": "2025-01-22T13-42-00-05-00_o1_gaia_merged.json",
78
- "gdm_intercode_ctf": "2025-01-22T20-46-35+00-00_o1_gdm-intercode-ctf_merged.json",
79
- "gdm_in_house_ctf": "2025-01-22T05-52-25+00-00_o1_gdm-in-house-ctf.json",
80
- "agentharm": "2025-01-21T09-05-42-08-00_agentharm_UGDq2yJeLAnPH6p7FgDgD8.json",
81
- "agentharm_benign": "2025-01-21T18-20-15-08-00_agentharm-benign_bkW2Bf5xLyDQdNtfLdjCpJ.json",
82
- "swe_bench": "2025-01-21T17-42-11+00-00_openai-o1.json"
83
- },
84
- "claude-3-5-sonnet-20241022": {
85
- "mmmu_multiple_choice": "2025-01-21T11-20-03-05-00_mmmu-multiple-choice_CWhKvGdoFo6pdHhDyi9GNm.json",
86
- "mmlu_pro": "2025-01-16T19-01-05-05-00_mmlu-pro_3vi84or97gQupuj5sT6vgZ.json",
87
- "hellaswag": "2025-01-15T15-09-33-05-00_hellaswag_QXqFxojvSToMu8ckHEMLkB.json",
88
- "gpqa_diamond": "2025-01-15T13-56-36-05-00_gpqa-diamond_eg4gFaMRENjnnYvQNtSB59.json",
89
- "gsm8k": "2025-01-15T14-23-25-05-00_gsm8k_nHB8Z4uZAwRAZFYpKmTptA.json",
90
- "mmmu_open": "2025-01-21T11-24-21-05-00_mmmu-open_SSjv3Dq9gZkEEUnvJUd5xf.json",
91
- "arc_easy": "2025-01-15T10-06-24-05-00_arc-easy_oBReQZQM5SAwMMD2jFshPb.json",
92
- "arc_challenge": "2025-01-15T10-12-11-05-00_arc-challenge_X8i6caCzkcQo5AT5zXkXso.json",
93
- "mmlu": "2025-01-16T15-16-51-05-00_mmlu_NFDs2kxmh3kQEbpbd8sz3w.json",
94
- "math": "2025-01-16T12-29-54-05-00_math_NvNQU58M8r3fpiwPGnvq8h.json",
95
- "ifeval": "2025-01-16T11-28-44-05-00_ifeval_fmWxch4ZjbmYCST6yUZsdV.json",
96
- "humaneval": "2025-01-16T11-26-12-05-00_humaneval_kUASiaNd9uZfWvCwYHhdF5.json",
97
- "winogrande": "2025-01-16T22-09-41-05-00_winogrande_mSWGAKg75E5RP79KWizvb9.json",
98
- "drop": "2025-01-15T10-15-15-05-00_drop_Z9A2Y84HYponNxnzNT9TNq.json",
99
- "gaia": "2025-01-12T23-57-37-05-00_claude-3-5-sonnet_gaia_merged.json",
100
- "gdm_intercode_ctf": "2025-01-11T02-47-45-05-00_claude-3-5-sonnet_gdm-intercode-ctf_merged.json",
101
- "gdm_in_house_ctf": "2025-01-11T07-41-14+00-00_claude-3-5-sonnet_gdm-in-house-ctf.json",
102
- "agentharm_benign": "2025-01-21T15-09-48-08-00_agentharm-benign_A3uBBWNvv88P5BsgqwFCfg.json",
103
- "agentharm": "2025-01-15T08-05-14-08-00_agentharm_VJGhWKLrVLdQczBZVgCXHc.json",
104
- "swe_bench": "2025-01-16T18-56-55+00-00_anthropic-claude-3-5-sonnet.json"
105
- },
106
- "c4ai-command-r-plus": {
107
- "ifeval": "2024-10-30T17-23-04-04-00_ifeval_RGucUMwdGmUnRpqyMTZTzW.json",
108
- "winogrande": "2024-10-30T14-42-18-04-00_winogrande_bY8yg7aRR5dCCK7NDCZEcc.json",
109
- "arc_challenge": "2024-10-29T17-30-03-04-00_arc-challenge_XB7LURXEGaxskWuLtYwdnW.json",
110
- "drop": "2024-10-30T12-06-30-04-00_drop_itY9cLiYAW2BF7NTeDceNd.json",
111
- "math": "2024-10-30T17-26-34-04-00_math_kohBUMpMFuMsR4jz4vUNWM.json",
112
- "gpqa_diamond": "2024-10-29T22-47-45-04-00_gpqa-diamond_JKpb6ya4pec9hh7uovPPCZ.json",
113
- "mmlu_pro": "2024-10-31T01-11-38-04-00_mmlu-pro_gZVAuy3zMKR23BieM5PqAX.json",
114
- "humaneval": "2024-10-30T17-22-23-04-00_humaneval_5ByPqUhoofSbKgvsUQNFCX.json",
115
- "gsm8k": "2024-10-30T15-03-35-04-00_gsm8k_QxbfbriJsKGQAg96JyjkoT.json",
116
- "hellaswag": "2024-10-30T15-18-17-04-00_hellaswag_UYyBTR6N8VJnKRmnbCrB8N.json",
117
- "mmlu": "2024-10-30T21-55-26-04-00_mmlu_JUPPLTzfe3Kme6UuorPTqg.json",
118
- "arc_easy": "2024-10-29T17-10-40-04-00_arc-easy_UvprihBMLXPF8JENVLRkdx.json"
119
- },
120
- "gpt-4o-mini": {
121
- "drop": "2024-10-30T17-36-25-04-00_drop_6TzJGqqEkpFUCxGD4QejV6.json",
122
- "humaneval": "2024-10-30T21-14-41-04-00_humaneval_Z9aXdUERuwYxoTheZ5GANC.json",
123
- "gpqa_diamond": "2024-10-30T19-31-26-04-00_gpqa-diamond_7aNe9wQiQKpNN96mfaWBPg.json",
124
- "mmmu_open": "2025-01-20T23-13-27-05-00_mmmu-open_GWi6XNYUSLq99BdabtScGm.json",
125
- "arc_challenge": "2024-10-30T17-34-51-04-00_arc-challenge_FbGgLswBZbRE4EhWiMyRt6.json",
126
- "mmlu": "2024-10-31T10-49-43-04-00_mmlu_oGb9mspeGbYS2gfbkknskN.json",
127
- "hellaswag": "2024-10-30T19-35-34-04-00_hellaswag_2SAz3cvMpDxFaApdHDR3s4.json",
128
- "ifeval": "2024-10-30T21-15-06-04-00_ifeval_nYs9KujQMQjcpbpbLtVx8G.json",
129
- "mmmu_multiple_choice": "2025-01-20T23-10-01-05-00_mmmu-multiple-choice_c5rLkrXkV83udX6DVJui5F.json",
130
- "arc_easy": "2024-10-30T17-29-56-04-00_arc-easy_XcEzqqPqJsRV29NqYDfnNo.json",
131
- "winogrande": "2024-10-31T00-59-07-04-00_winogrande_Ci55vHvbGGW38zVpMCwtWa.json",
132
- "mmlu_pro": "2024-10-30T22-43-30-04-00_mmlu-pro_Dc2uu3EV7MJtjg6gg5Y9qH.json",
133
- "math": "2024-10-30T21-15-49-04-00_math_YsWdRzpqMq2dqQ9SPKfack.json",
134
- "gsm8k": "2024-10-30T19-32-39-04-00_gsm8k_nLSssETKDDWNktAFWnVwfv.json"
135
- },
136
- "Meta-Llama-3.1-70B-Instruct": {
137
- "hellaswag": "2024-10-30T00-45-54-04-00_hellaswag_BKfQG9yGAr383MGnooMLBH.json",
138
- "drop": "2024-10-29T21-01-02-04-00_drop_LzAWvLWkNrNKu5qf56wXRo.json",
139
- "gpqa_diamond": "2024-10-29T23-41-39-04-00_gpqa-diamond_TdLdYmVM6GCVMAECcXkuhj.json",
140
- "winogrande": "2024-10-30T09-20-56-04-00_winogrande_WnUgkSRhSMvh3zUjnuJWQZ.json",
141
- "gsm8k": "2024-10-30T00-03-31-04-00_gsm8k_bKsUfCAfcmBCeryboNaLoX.json",
142
- "math": "2024-10-30T02-34-50-04-00_math_2xiNcrGih26uzJdG4q88bM.json",
143
- "ifeval": "2024-10-30T02-29-32-04-00_ifeval_Dwh3CF2ZYFrvw7UcTwrsvK.json",
144
- "arc_challenge": "2024-10-29T20-58-56-04-00_arc-challenge_oFL5wFjT7KwNFhMFfe72JN.json",
145
- "arc_easy": "2024-10-29T20-53-12-04-00_arc-easy_UXzR7cDeNteP39NoXUYnhm.json",
146
- "mmlu_pro": "2024-10-30T06-11-16-04-00_mmlu-pro_oQiEBJdeKtEEt4cm9KL7uy.json",
147
- "humaneval": "2024-10-30T02-28-25-04-00_humaneval_KcJV2rHuHJ2JLxijihEkcW.json",
148
- "mmlu": "2024-10-30T03-51-50-04-00_mmlu_6SNjs2QmPRvqGnvbnNtaqb.json"
149
- },
150
- "gpt-4o": {
151
- "gpqa_diamond": "2024-10-31T03-29-33-04-00_gpqa-diamond_nFmRv5MJiYjHjezmq4V6Va.json",
152
- "arc_challenge": "2024-10-31T01-45-55-04-00_arc-challenge_nrsPPxh4DpzgLPQDFdcfVp.json",
153
- "gsm8k": "2024-10-31T03-31-16-04-00_gsm8k_jVXeSvHowbietZCFsFYCwB.json",
154
- "mmlu": "2024-10-31T10-49-43-04-00_mmlu_GarLpfQFSpM3C22nbbGp54.json",
155
- "ifeval": "2024-10-31T05-00-11-04-00_ifeval_jxreUu8JqRdkrcHP4E3hLR.json",
156
- "mmlu_pro": "2024-10-31T06-59-42-04-00_mmlu-pro_EuAKDwAWSfNVpqyyqrf2Ba.json",
157
- "mmmu_open": "2025-01-20T23-07-46-05-00_mmmu-open_d3Q2HvuPZzEX6FAM4NBhnp.json",
158
- "winogrande": "2024-10-31T09-02-03-04-00_winogrande_44kKF7M9mKoqVC7ixZVXuq.json",
159
- "drop": "2024-10-31T01-47-20-04-00_drop_3gxDcn6vUoR3nvHX9BcSq4.json",
160
- "arc_easy": "2024-10-31T01-41-34-04-00_arc-easy_nUavRHdiRVfrxo6dmCPadh.json",
161
- "mmmu_multiple_choice": "2025-01-20T23-03-21-05-00_mmmu-multiple-choice_eoycAFLMirSqiURdXmBP2e.json",
162
- "humaneval": "2024-10-31T04-59-42-04-00_humaneval_nmJcd84CcNKjWS8fBfMbZM.json",
163
- "math": "2024-10-31T05-01-22-04-00_math_cDSpKPp3nLrFy8uYfYKEbM.json",
164
- "hellaswag": "2024-10-31T03-33-47-04-00_hellaswag_JNnnPuz3dhZRpyXzizMUBF.json",
165
- "gaia": "2025-01-13T15-53-22+00-00_gpt-4o_gaia_merged.json",
166
- "gdm_intercode_ctf": "2025-01-08T10-06-29-05-00_gpt-4o_gdm-intercode-ctf_merged.json",
167
- "gdm_in_house_ctf": "2025-01-11T07-02-14+00-00_gpt-4o_gdm-in-house-ctf.json",
168
- "agentharm": "2025-01-07T16-34-15-08-00_agentharm_UfSoyHEAH2E5RVdrPVUemy.json",
169
- "agentharm_benign": "2025-01-21T13-45-18-08-00_agentharm-benign_8DhGJqEAvw6o8uCv4a4dVz.json",
170
- "swe_bench": "2025-01-14T23-09-10+00-00_openai-gpt-4o.json"
171
- },
172
- "Mistral-Large-Instruct-2407": {
173
- "drop": "2024-10-31T01-56-12-04-00_drop_NtvuCoU2LoMbH8DztcCTen.json",
174
- "ifeval": "2024-10-31T06-30-16-04-00_ifeval_TLkvCSFEWo4PLv6hAha7YB.json",
175
- "mmlu": "2024-10-31T07-21-48-04-00_mmlu_YnUhmHoStr3WuJdchWmNPt.json",
176
- "gpqa_diamond": "2024-10-31T04-22-52-04-00_gpqa-diamond_SuZUZxGdqS2ZecbLRNkKd4.json",
177
- "gsm8k": "2024-10-31T04-28-49-04-00_gsm8k_5tQp9tbwUMj6NpjNKCAfVm.json",
178
- "math": "2024-10-31T06-33-09-04-00_math_2CmjBedAfUxqvmcHRdBgyB.json",
179
- "arc_easy": "2024-10-31T01-48-39-04-00_arc-easy_YbfuBT3usZXt2xgZkkR5dq.json",
180
- "mmlu_pro": "2024-10-31T09-41-25-04-00_mmlu-pro_fyYT4aabPesfY5TpzFMPnd.json",
181
- "humaneval": "2024-10-31T06-29-24-04-00_humaneval_nu8SUSGekKJWB8HLKDigYK.json",
182
- "hellaswag": "2024-10-31T04-50-00-04-00_hellaswag_ZzQoZ6gkRQsTzMhQr7GYNn.json",
183
- "arc_challenge": "2024-10-31T01-54-13-04-00_arc-challenge_WfQRhMkFcywefpU46isBVP.json",
184
- "winogrande": "2024-10-31T11-57-58-04-00_winogrande_TP3UGwpp37Dyv6ks9Ty5Hk.json"
185
- },
186
- "Qwen2.5-72B-Instruct": {
187
- "arc_challenge": "2024-10-31T13-46-34-04-00_arc-challenge_FSybKYYwpXVLQag8VwpjKe.json",
188
- "mmlu_pro": "2024-11-01T20-31-04-04-00_mmlu-pro_2TfSPmsVmKatntHy2CnR7A.json",
189
- "gpqa_diamond": "2024-10-31T13-48-32-04-00_gpqa-diamond_8qSySicySUyNvRRYVFBKLU.json",
190
- "winogrande": "2024-10-31T14-46-29-04-00_winogrande_CX692dYh53gJ6JigT9GMpa.json",
191
- "mmlu": "2024-11-01T10-08-50-04-00_mmlu_AgK27yYvmAo2LxotBH7ZL9.json",
192
- "hellaswag": "2024-11-01T02-55-55-04-00_hellaswag_RSk8rGcQWg3HRrLffTNoiM.json",
193
- "gsm8k": "2024-11-01T01-15-16-04-00_gsm8k_3h4W6xZjXpz9oCwtgKNYzo.json",
194
- "arc_easy": "2024-10-31T13-40-08-04-00_arc-easy_3JUyzfoEHxhSBUdCU2AaVC.json",
195
- "math": "2024-11-01T10-06-46-04-00_math_UUpS2R9eQc9KxBxkanT2gE.json",
196
- "ifeval": "2024-10-31T14-51-45-04-00_ifeval_VGxA7gTZLZSruceM9Ci37C.json",
197
- "humaneval": "2024-10-31T14-49-39-04-00_humaneval_9u7khnxivCDroJoPNRFpjs.json",
198
- "drop": "2024-10-31T15-03-20-04-00_drop_DDLi98VhiV2bLzuw7fx6H4.json"
199
- }
200
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
data/populate_results.py DELETED
@@ -1,41 +0,0 @@
1
- import json
2
-
3
- def get_log_url(model_name: str, log_file_name: str) -> str:
4
- """Returns the URL to the log file for a given model and benchmark"""
5
- if log_file_name is None:
6
- return None
7
- else:
8
- # replace .json with .eval
9
- log_file_name = log_file_name.replace(".json", ".eval")
10
- return f"https://storage.googleapis.com/inspect-evals/eval/{model_name}/index.html?log_file=logs/logs/{log_file_name}"
11
-
12
- def main():
13
- # Load the results and log file names
14
- with open("data/results.json", "r") as f:
15
- results = json.load(f)
16
-
17
- with open("data/inspect_log_file_names.json", "r") as f:
18
- log_files = json.load(f)
19
-
20
- # For each model in results
21
- for model_name, model_data in results.items():
22
- # Get the log files for this model
23
- model_logs = log_files.get(model_name, {})
24
-
25
- # For each task in the model's results
26
- for task_name, task_data in model_data["results"].items():
27
- # Get the log file name for this task
28
- log_file_name = model_logs.get(task_name)
29
-
30
- # Add the log URL to the task data
31
- if log_file_name:
32
- task_data["log_url"] = get_log_url(model_name, log_file_name)
33
- else:
34
- task_data["log_url"] = None
35
-
36
- # Save the updated results
37
- with open("data/results_with_logs.json", "w") as f:
38
- json.dump(results, f, indent=4)
39
-
40
- if __name__ == "__main__":
41
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
data/results.json.bak DELETED
@@ -1,760 +0,0 @@
1
- {
2
- "DeepSeek-R1": {
3
- "config": {
4
- "model_name": "DeepSeek-R1",
5
- "model_sha": "https://api-docs.deepseek.com/news/news250120",
6
- "model_dtype": "torch.float16"
7
- },
8
- "results": {
9
- "mmlu_pro": {
10
- "accuracy": 0.8382646276595744
11
- },
12
- "humaneval": {
13
- "mean": 0.9567901234567902
14
- },
15
- "math": {
16
- "accuracy": 0.9272
17
- },
18
- "gsm8k": {
19
- "accuracy": 0.954510993176649
20
- },
21
- "arc_challenge": {
22
- "accuracy": 0.9667235494880546
23
- },
24
- "winogrande": {
25
- "accuracy": 0.9179163378058406
26
- },
27
- "arc_easy": {
28
- "accuracy": 0.9873737373737373
29
- },
30
- "gpqa_diamond": {
31
- "accuracy": 0.7045454545454546
32
- },
33
- "drop": {
34
- "mean": null
35
- },
36
- "hellaswag": {
37
- "accuracy": null
38
- },
39
- "ifeval": {
40
- "final_acc": null
41
- },
42
- "mmlu": {
43
- "accuracy": null
44
- },
45
- "mmmu_multiple_choice": {
46
- "accuracy": null
47
- },
48
- "mmmu_open": {
49
- "accuracy": null
50
- },
51
- "gaia": {
52
- "accuracy": null
53
- },
54
- "gdm_intercode_ctf": {
55
- "accuracy": null
56
- },
57
- "gdm_in_house_ctf": {
58
- "accuracy": null
59
- },
60
- "agentharm": {
61
- "avg_score": null
62
- },
63
- "agentharm_benign": {
64
- "avg_score": null
65
- },
66
- "swe_bench": {
67
- "mean": null
68
- }
69
- }
70
- },
71
- "Meta-Llama-3.1-70B-Instruct": {
72
- "config": {
73
- "model_name": "Meta-Llama-3.1-70B-Instruct",
74
- "model_sha": "https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct",
75
- "model_dtype": "torch.float16"
76
- },
77
- "results": {
78
- "hellaswag": {
79
- "accuracy": 0.869946225851424
80
- },
81
- "drop": {
82
- "mean": 0.8811263765076035
83
- },
84
- "gpqa_diamond": {
85
- "accuracy": 0.4318181818181818
86
- },
87
- "winogrande": {
88
- "accuracy": 0.8666140489344909
89
- },
90
- "gsm8k": {
91
- "accuracy": 0.9469294920394238
92
- },
93
- "math": {
94
- "accuracy": 0.6004
95
- },
96
- "ifeval": {
97
- "final_acc": 0.8604907201780166
98
- },
99
- "arc_challenge": {
100
- "accuracy": 0.9445392491467577
101
- },
102
- "arc_easy": {
103
- "accuracy": 0.9823232323232324
104
- },
105
- "mmlu_pro": {
106
- "accuracy": 0.6688829787234043
107
- },
108
- "humaneval": {
109
- "mean": 0.7865853658536586
110
- },
111
- "mmlu": {
112
- "accuracy": 0.8033755875231449
113
- },
114
- "mmmu_multiple_choice": {
115
- "accuracy": null
116
- },
117
- "mmmu_open": {
118
- "accuracy": null
119
- },
120
- "gaia": {
121
- "accuracy": null
122
- },
123
- "gdm_intercode_ctf": {
124
- "accuracy": null
125
- },
126
- "gdm_in_house_ctf": {
127
- "accuracy": null
128
- },
129
- "agentharm": {
130
- "avg_score": null
131
- },
132
- "agentharm_benign": {
133
- "avg_score": null
134
- },
135
- "swe_bench": {
136
- "mean": null
137
- }
138
- }
139
- },
140
- "Mistral-Large-Instruct-2407": {
141
- "config": {
142
- "model_name": "Mistral-Large-Instruct-2407",
143
- "model_sha": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407",
144
- "model_dtype": "torch.float16"
145
- },
146
- "results": {
147
- "drop": {
148
- "mean": 0.7424257996853698
149
- },
150
- "ifeval": {
151
- "final_acc": 0.8285172231900246
152
- },
153
- "mmlu": {
154
- "accuracy": 0.8035892323030908
155
- },
156
- "gpqa_diamond": {
157
- "accuracy": 0.4734848484848485
158
- },
159
- "gsm8k": {
160
- "accuracy": 0.9378316906747536
161
- },
162
- "math": {
163
- "accuracy": 0.6574
164
- },
165
- "arc_easy": {
166
- "accuracy": 0.9852693602693603
167
- },
168
- "mmlu_pro": {
169
- "accuracy": 0.6942320478723404
170
- },
171
- "humaneval": {
172
- "mean": 0.8658536585365854
173
- },
174
- "hellaswag": {
175
- "accuracy": 0.9047998406691894
176
- },
177
- "arc_challenge": {
178
- "accuracy": 0.9436860068259386
179
- },
180
- "winogrande": {
181
- "accuracy": 0.8547750591949487
182
- },
183
- "mmmu_multiple_choice": {
184
- "accuracy": null
185
- },
186
- "mmmu_open": {
187
- "accuracy": null
188
- },
189
- "gaia": {
190
- "accuracy": null
191
- },
192
- "gdm_intercode_ctf": {
193
- "accuracy": null
194
- },
195
- "gdm_in_house_ctf": {
196
- "accuracy": null
197
- },
198
- "agentharm": {
199
- "avg_score": null
200
- },
201
- "agentharm_benign": {
202
- "avg_score": null
203
- },
204
- "swe_bench": {
205
- "mean": null
206
- }
207
- }
208
- },
209
- "c4ai-command-r-plus": {
210
- "config": {
211
- "model_name": "c4ai-command-r-plus",
212
- "model_sha": "https://huggingface.co/CohereForAI/c4ai-command-r-plus"
213
- },
214
- "results": {
215
- "ifeval": {
216
- "final_acc": 0.7779591483929307
217
- },
218
- "winogrande": {
219
- "accuracy": 0.7490134175217048
220
- },
221
- "arc_challenge": {
222
- "accuracy": 0.8506825938566553
223
- },
224
- "drop": {
225
- "mean": 0.743557420031463
226
- },
227
- "math": {
228
- "accuracy": 0.2626
229
- },
230
- "gpqa_diamond": {
231
- "accuracy": 0.3194444444444444
232
- },
233
- "mmlu_pro": {
234
- "accuracy": 0.441156914893617
235
- },
236
- "humaneval": {
237
- "mean": 0.6219512195121951
238
- },
239
- "gsm8k": {
240
- "accuracy": 0.7816527672479151
241
- },
242
- "hellaswag": {
243
- "accuracy": 0.7954590718980283
244
- },
245
- "mmlu": {
246
- "accuracy": 0.695128899017234
247
- },
248
- "arc_easy": {
249
- "accuracy": 0.9377104377104377
250
- },
251
- "mmmu_multiple_choice": {
252
- "accuracy": null
253
- },
254
- "mmmu_open": {
255
- "accuracy": null
256
- },
257
- "gaia": {
258
- "accuracy": null
259
- },
260
- "gdm_intercode_ctf": {
261
- "accuracy": null
262
- },
263
- "gdm_in_house_ctf": {
264
- "accuracy": null
265
- },
266
- "agentharm": {
267
- "avg_score": null
268
- },
269
- "agentharm_benign": {
270
- "avg_score": null
271
- },
272
- "swe_bench": {
273
- "mean": null
274
- }
275
- }
276
- },
277
- "claude-3-5-sonnet-20241022": {
278
- "config": {
279
- "model_name": "claude-3-5-sonnet-20241022",
280
- "model_sha": "https://www.anthropic.com/claude/sonnet",
281
- "model_dtype": "torch.float16"
282
- },
283
- "results": {
284
- "mmmu_multiple_choice": {
285
- "accuracy": 0.6481700118063755
286
- },
287
- "mmlu_pro": {
288
- "accuracy": 0.7762632978723404
289
- },
290
- "hellaswag": {
291
- "accuracy": 0.9228241386178052
292
- },
293
- "gpqa_diamond": {
294
- "accuracy": 0.6098484848484849
295
- },
296
- "gsm8k": {
297
- "accuracy": 0.9620924943138741
298
- },
299
- "mmmu_open": {
300
- "accuracy": 0.41509433962264153
301
- },
302
- "arc_easy": {
303
- "accuracy": 0.9915824915824916
304
- },
305
- "arc_challenge": {
306
- "accuracy": 0.9692832764505119
307
- },
308
- "mmlu": {
309
- "accuracy": 0.8665432274604757
310
- },
311
- "math": {
312
- "accuracy": 0.7942
313
- },
314
- "ifeval": {
315
- "final_acc": 0.8958114469607309
316
- },
317
- "humaneval": {
318
- "mean": 0.9451219512195121
319
- },
320
- "winogrande": {
321
- "accuracy": 0.9021310181531176
322
- },
323
- "drop": {
324
- "mean": 0.8977608809648663
325
- },
326
- "gaia": {
327
- "accuracy": 0.3381818181818182
328
- },
329
- "gdm_intercode_ctf": {
330
- "accuracy": 0.8556962025316455
331
- },
332
- "gdm_in_house_ctf": {
333
- "accuracy": 0.6153846153846154
334
- },
335
- "agentharm": {
336
- "avg_score": 0.14767992424242424
337
- },
338
- "agentharm_benign": {
339
- "avg_score": 0.800704570051161
340
- },
341
- "swe_bench": {
342
- "mean": 0.0672
343
- }
344
- }
345
- },
346
- "gemini-1.5-flash": {
347
- "config": {
348
- "model_name": "gemini-1.5-flash",
349
- "model_sha": "https://deepmind.google/technologies/gemini/flash",
350
- "model_dtype": "torch.float16"
351
- },
352
- "results": {
353
- "gpqa_diamond": {
354
- "accuracy": 0.40404040404040403
355
- },
356
- "arc_challenge": {
357
- "accuracy": 0.9308873720136519
358
- },
359
- "math": {
360
- "accuracy": 0.452
361
- },
362
- "mmmu_open": {
363
- "accuracy": 0.16981132075471697
364
- },
365
- "drop": {
366
- "mean": 0.751044572627163
367
- },
368
- "mmlu_pro": {
369
- "accuracy": 0.5993184840425532
370
- },
371
- "ifeval": {
372
- "final_acc": 0.7681296737102001
373
- },
374
- "hellaswag": {
375
- "accuracy": 0.8557060346544513
376
- },
377
- "winogrande": {
378
- "accuracy": 0.7884767166535123
379
- },
380
- "humaneval": {
381
- "mean": 0.7439024390243902
382
- },
383
- "arc_easy": {
384
- "accuracy": 0.984006734006734
385
- },
386
- "gsm8k": {
387
- "accuracy": 0.8582259287338894
388
- },
389
- "mmlu": {
390
- "accuracy": 0.7714713003845606
391
- },
392
- "mmmu_multiple_choice": {
393
- "accuracy": 0.5702479338842975
394
- },
395
- "gaia": {
396
- "accuracy": null
397
- },
398
- "gdm_intercode_ctf": {
399
- "accuracy": null
400
- },
401
- "gdm_in_house_ctf": {
402
- "accuracy": null
403
- },
404
- "agentharm": {
405
- "avg_score": null
406
- },
407
- "agentharm_benign": {
408
- "avg_score": null
409
- },
410
- "swe_bench": {
411
- "mean": null
412
- }
413
- }
414
- },
415
- "gemini-1.5-pro": {
416
- "config": {
417
- "model_name": "gemini-1.5-pro",
418
- "model_sha": "https://deepmind.google/technologies/gemini/pro",
419
- "model_dtype": "torch.float16"
420
- },
421
- "results": {
422
- "mmlu": {
423
- "accuracy": 0.8467454778521578
424
- },
425
- "humaneval": {
426
- "mean": 0.8719512195121951
427
- },
428
- "mmmu_multiple_choice": {
429
- "accuracy": 0.6304604486422668
430
- },
431
- "mmlu_pro": {
432
- "accuracy": 0.7563996010638298
433
- },
434
- "math": {
435
- "accuracy": 0.852
436
- },
437
- "arc_easy": {
438
- "accuracy": 0.9877946127946128
439
- },
440
- "mmmu_open": {
441
- "accuracy": 0.3584905660377358
442
- },
443
- "gsm8k": {
444
- "accuracy": 0.9613343442001516
445
- },
446
- "gpqa_diamond": {
447
- "accuracy": 0.5782828282828283
448
- },
449
- "ifeval": {
450
- "final_acc": 0.8982344623377084
451
- },
452
- "winogrande": {
453
- "accuracy": 0.8768745067087609
454
- },
455
- "arc_challenge": {
456
- "accuracy": 0.9633105802047781
457
- },
458
- "drop": {
459
- "mean": 0.8800912427897221
460
- },
461
- "hellaswag": {
462
- "accuracy": 0.9123680541724756
463
- },
464
- "gaia": {
465
- "accuracy": 0.13818181818181818
466
- },
467
- "gdm_intercode_ctf": {
468
- "accuracy": 0.5291139240506328
469
- },
470
- "gdm_in_house_ctf": {
471
- "accuracy": 0.23076923076923078
472
- },
473
- "agentharm": {
474
- "avg_score": 0.2898649645808737
475
- },
476
- "agentharm_benign": {
477
- "avg_score": 0.5961489079102715
478
- },
479
- "swe_bench": {
480
- "mean": 0.004
481
- }
482
- }
483
- },
484
- "gpt-4o": {
485
- "config": {
486
- "model_name": "gpt-4o",
487
- "model_sha": "https://openai.com/index/hello-gpt-4o",
488
- "model_dtype": "torch.float16"
489
- },
490
- "results": {
491
- "gpqa_diamond": {
492
- "accuracy": 0.51010101010101
493
- },
494
- "arc_challenge": {
495
- "accuracy": 0.9633105802047781
496
- },
497
- "gsm8k": {
498
- "accuracy": 0.9446550416982562
499
- },
500
- "mmlu": {
501
- "accuracy": 0.8435408061529697
502
- },
503
- "ifeval": {
504
- "final_acc": 0.8780386042367585
505
- },
506
- "mmlu_pro": {
507
- "accuracy": 0.7450964095744681
508
- },
509
- "mmmu_open": {
510
- "accuracy": 0.3584905660377358
511
- },
512
- "winogrande": {
513
- "accuracy": 0.9013417521704814
514
- },
515
- "drop": {
516
- "mean": 0.7511693759832198
517
- },
518
- "arc_easy": {
519
- "accuracy": 0.9915824915824916
520
- },
521
- "mmmu_multiple_choice": {
522
- "accuracy": 0.5903187721369539
523
- },
524
- "humaneval": {
525
- "mean": 0.9085365853658537
526
- },
527
- "math": {
528
- "accuracy": 0.7054
529
- },
530
- "hellaswag": {
531
- "accuracy": 0.924317864967138
532
- },
533
- "gaia": {
534
- "accuracy": 0.16606060606060608
535
- },
536
- "gdm_intercode_ctf": {
537
- "accuracy": 0.6379746835443038
538
- },
539
- "gdm_in_house_ctf": {
540
- "accuracy": 0.23076923076923078
541
- },
542
- "agentharm": {
543
- "avg_score": 0.49953844451003543
544
- },
545
- "agentharm_benign": {
546
- "avg_score": 0.8249433048012594
547
- },
548
- "swe_bench": {
549
- "mean": 0.012
550
- }
551
- }
552
- },
553
- "gpt-4o-mini": {
554
- "config": {
555
- "model_name": "gpt-4o-mini",
556
- "model_sha": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence",
557
- "model_dtype": "torch.float16"
558
- },
559
- "results": {
560
- "drop": {
561
- "mean": 0.8065915049816466
562
- },
563
- "humaneval": {
564
- "mean": 0.8597560975609756
565
- },
566
- "gpqa_diamond": {
567
- "accuracy": 0.3838383838383838
568
- },
569
- "mmmu_open": {
570
- "accuracy": 0.18867924528301888
571
- },
572
- "arc_challenge": {
573
- "accuracy": 0.9249146757679181
574
- },
575
- "mmlu": {
576
- "accuracy": 0.7698333570716422
577
- },
578
- "hellaswag": {
579
- "accuracy": 0.8750248954391555
580
- },
581
- "ifeval": {
582
- "final_acc": 0.8419061423689144
583
- },
584
- "mmmu_multiple_choice": {
585
- "accuracy": 0.5395513577331759
586
- },
587
- "arc_easy": {
588
- "accuracy": 0.9793771043771043
589
- },
590
- "winogrande": {
591
- "accuracy": 0.7529597474348856
592
- },
593
- "mmlu_pro": {
594
- "accuracy": 0.6396276595744681
595
- },
596
- "math": {
597
- "accuracy": 0.633
598
- },
599
- "gsm8k": {
600
- "accuracy": 0.9181197877179682
601
- },
602
- "gaia": {
603
- "accuracy": null
604
- },
605
- "gdm_intercode_ctf": {
606
- "accuracy": null
607
- },
608
- "gdm_in_house_ctf": {
609
- "accuracy": null
610
- },
611
- "agentharm": {
612
- "avg_score": null
613
- },
614
- "agentharm_benign": {
615
- "avg_score": null
616
- },
617
- "swe_bench": {
618
- "mean": null
619
- }
620
- }
621
- },
622
- "o1": {
623
- "config": {
624
- "model_name": "o1",
625
- "model_sha": "https://openai.com/o1",
626
- "model_dtype": "torch.float16"
627
- },
628
- "results": {
629
- "winogrande": {
630
- "accuracy": 0.9392265193370166
631
- },
632
- "humaneval": {
633
- "mean": 0.9695121951219512
634
- },
635
- "mmmu_open": {
636
- "accuracy": 0.6981132075471698
637
- },
638
- "math": {
639
- "accuracy": 0.959
640
- },
641
- "arc_easy": {
642
- "accuracy": 0.9911616161616161
643
- },
644
- "arc_challenge": {
645
- "accuracy": 0.9786689419795221
646
- },
647
- "gsm8k": {
648
- "accuracy": 0.9416224412433661
649
- },
650
- "gpqa_diamond": {
651
- "accuracy": 0.7550505050505051
652
- },
653
- "mmlu_pro": {
654
- "accuracy": 0.8447473404255319
655
- },
656
- "mmmu_multiple_choice": {
657
- "accuracy": 0.8063754427390791
658
- },
659
- "drop": {
660
- "mean": null
661
- },
662
- "hellaswag": {
663
- "accuracy": null
664
- },
665
- "ifeval": {
666
- "final_acc": null
667
- },
668
- "mmlu": {
669
- "accuracy": null
670
- },
671
- "gaia": {
672
- "accuracy": 0.41090909090909084
673
- },
674
- "gdm_intercode_ctf": {
675
- "accuracy": 0.8481012658227849
676
- },
677
- "gdm_in_house_ctf": {
678
- "accuracy": 0.46153846153846156
679
- },
680
- "agentharm": {
681
- "avg_score": 0.08782061688311688
682
- },
683
- "agentharm_benign": {
684
- "avg_score": 0.7235176849665487
685
- },
686
- "swe_bench": {
687
- "mean": 0.0036
688
- }
689
- }
690
- },
691
- "o3-mini": {
692
- "config": {
693
- "model_name": "o3-mini",
694
- "model_sha": "https://openai.com/index/openai-o3-mini",
695
- "model_dtype": "torch.float16"
696
- },
697
- "results": {
698
- "math": {
699
- "accuracy": 0.9691320905993185
700
- },
701
- "humaneval": {
702
- "mean": 0.9817073170731707
703
- },
704
- "mmlu_pro": {
705
- "accuracy": 0.7924606807023383
706
- },
707
- "gpqa_diamond": {
708
- "accuracy": 0.7365319865319865
709
- },
710
- "winogrande": {
711
- "accuracy": 0.8492501973164956
712
- },
713
- "gsm8k": {
714
- "accuracy": 0.9454131918119788
715
- },
716
- "arc_challenge": {
717
- "accuracy": 0.9641638225255973
718
- },
719
- "arc_easy": {
720
- "accuracy": 0.9755892255892256
721
- },
722
- "drop": {
723
- "mean": null
724
- },
725
- "hellaswag": {
726
- "accuracy": null
727
- },
728
- "ifeval": {
729
- "final_acc": null
730
- },
731
- "mmlu": {
732
- "accuracy": null
733
- },
734
- "mmmu_multiple_choice": {
735
- "accuracy": null
736
- },
737
- "mmmu_open": {
738
- "accuracy": null
739
- },
740
- "gaia": {
741
- "accuracy": 0.27030303030303043
742
- },
743
- "gdm_intercode_ctf": {
744
- "accuracy": 0.8278481012658225
745
- },
746
- "gdm_in_house_ctf": {
747
- "accuracy": 0.38461538461538464
748
- },
749
- "agentharm": {
750
- "avg_score": 0.1241931080283353
751
- },
752
- "agentharm_benign": {
753
- "avg_score": 0.5429306867375049
754
- },
755
- "swe_bench": {
756
- "mean": 0.0024
757
- }
758
- }
759
- }
760
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
refactor_eval_results.py DELETED
@@ -1,192 +0,0 @@
1
- import json
2
- import os
3
-
4
-
5
- METRIC_NAME = {
6
- # single-turn
7
- "arc_easy": "accuracy",
8
- "arc_challenge": "accuracy",
9
- "gpqa_diamond": "accuracy",
10
- "drop": "mean",
11
- "winogrande": "accuracy",
12
- "gsm8k": "accuracy",
13
- "hellaswag": "accuracy",
14
- "humaneval": "mean",
15
- "ifeval": "final_acc",
16
- "math": "accuracy",
17
- "mmlu": "accuracy",
18
- "mmlu_pro": "accuracy",
19
- "mmmu_multiple_choice": "accuracy",
20
- "mmmu_open": "accuracy",
21
-
22
- # agentic
23
- "gaia": "accuracy",
24
- "gdm_intercode_ctf": "accuracy",
25
- "gdm_in_house_ctf": "accuracy",
26
- "agentharm": "avg_score",
27
- "agentharm_benign": "avg_score",
28
- "swe_bench": "mean",
29
- }
30
-
31
- MODEL_SHA_MAP = {
32
- # open source models
33
- "c4ai-command-r-plus": "https://huggingface.co/CohereForAI/c4ai-command-r-plus",
34
- "Meta-Llama-3.1-70B-Instruct": "https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct",
35
- "Mistral-Large-Instruct-2407": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407",
36
- "Qwen2.5-72B-Instruct": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct",
37
-
38
- # closed source models
39
- "claude-3-5-sonnet-20241022": "https://www.anthropic.com/claude/sonnet",
40
- "gemini-1.5-flash": "https://deepmind.google/technologies/gemini/flash", # TODO: points to 2.0, can't find page for 1.5
41
- "gemini-1.5-pro": "https://deepmind.google/technologies/gemini/pro",
42
- "gpt-4o": "https://openai.com/index/hello-gpt-4o",
43
- "gpt-4o-mini": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence",
44
- "o1": "https://openai.com/o1",
45
- "o3-mini": "https://openai.com/index/openai-o3-mini",
46
- "DeepSeek-R1": "https://api-docs.deepseek.com/news/news250120"
47
- }
48
-
49
- MODEL_VERSION_MAP = {
50
- # open source models
51
- "c4ai-command-r-plus": "c4ai-command-r-plus",
52
- "Meta-Llama-3.1-70B-Instruct": "Llama-3.1-70B-Instruct",
53
- "Mistral-Large-Instruct-2407": "Mistral-Large-Instruct-2407",
54
- "Qwen2.5-72B-Instruct": "Qwen2.5-72B-Instruct",
55
-
56
- # closed source models
57
- "claude-3-5-sonnet-20241022": "Claude-3.5-Sonnet-20241022",
58
- "gemini-1.5-flash": "Gemini-1.5-Flash",
59
- "gemini-1.5-pro": "Gemini-1.5-Pro-002",
60
- "gpt-4o": "GPT-4o-20240806",
61
- "gpt-4o-mini": "GPT-4o-mini-20240718",
62
- "o1": "o1-20241217",
63
- "o3-mini": "o3-mini-20250131",
64
- "DeepSeek-R1": "DeepSeek-R1",
65
- }
66
-
67
- AGENTIC_LOG_MODEL_NAME_MAP = {
68
- "claude-3-5-sonnet-20241022": "claude-3-5-sonnet-20241022",
69
- "gemini-1.5-pro": "gemini-1.5-pro-002",
70
- "gpt-4o": "gpt-4o-2024-08-06",
71
- "o1": "o1-2024-12-17",
72
- "o3-mini": "o3-mini-2025-01-31",
73
- }
74
-
75
- AGENTIC_TASKS = ["gaia", "gdm-intercode-ctf", "gdm-in-house-ctf", "agentharm", "swe-bench"]
76
-
77
-
78
- def combine_eval_results(results_path: str, model_name: str, type: str,) -> dict:
79
- results = dict(
80
- {
81
- "config": {
82
- "model_name": model_name,
83
- # dummy keys
84
- "model_sha": MODEL_SHA_MAP[model_name],
85
- "model_dtype": "torch.float16",
86
- },
87
- "results": {},
88
- }
89
- )
90
-
91
- if type == "base":
92
- for file in os.listdir(os.path.join(results_path, model_name)):
93
- if file.endswith(".json"):
94
- with open(os.path.join(results_path, model_name, file), "r") as f:
95
- try:
96
- result = json.load(f)
97
- task_name = result["eval"]["task"].split("/")[-1]
98
- if task_name == "math":
99
- metrics = [elm for elm in result["results"]["scores"] if elm["name"] == "expression_equivalance"][0]["metrics"] # TODO: change scorer if required
100
- else:
101
- metrics = result["results"]["scores"][0]["metrics"]
102
- metric_name = metrics[METRIC_NAME[task_name]]["name"]
103
- metric_value = metrics[METRIC_NAME[task_name]]["value"]
104
- results["results"].update(
105
- {
106
- task_name: {
107
- metric_name: metric_value
108
- }
109
- }
110
- )
111
- except KeyError as e:
112
- print(f"KeyError: {e}")
113
- print(model_name)
114
- print(file)
115
-
116
- elif type == "agentic":
117
- model_name = AGENTIC_LOG_MODEL_NAME_MAP[model_name] # change name based on log file structure
118
- results_path = os.path.join(results_path, model_name)
119
- for task in AGENTIC_TASKS:
120
- for file in os.listdir(os.path.join(results_path, task)):
121
- if file.endswith(".json"):
122
- with open(os.path.join(results_path, task, file), "r") as f:
123
- try:
124
- result = json.load(f)
125
- task_name = result["eval"]["task"].split("/")[-1]
126
- metrics = result["results"]["scores"][0]["metrics"]
127
- metric_name = metrics[METRIC_NAME[task_name]]["name"].split("/")[-1]
128
- metric_value = metrics[METRIC_NAME[task_name]]["value"]
129
- results["results"].update(
130
- {
131
- task_name: {
132
- metric_name: metric_value
133
- }
134
- }
135
- )
136
- except KeyError as e:
137
- print(f"KeyError: {e}")
138
- print(model_name)
139
- print(file)
140
-
141
- return results
142
-
143
-
144
- def main():
145
-
146
- CACHE_PATH=os.getenv("HF_HOME", ".")
147
- EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
148
- EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
149
-
150
- base_bm_input_path = "./base_benchmarking_logs"
151
- agentic_bm_input_path = "/fs01/projects/aieng/public/inspect_evals/agentic_benchmarking_runs"
152
- os.makedirs(EVAL_RESULTS_PATH, exist_ok=True)
153
- os.makedirs(EVAL_REQUESTS_PATH, exist_ok=True)
154
-
155
- for model_name in os.listdir(base_bm_input_path):
156
-
157
- if os.path.isdir(os.path.join(base_bm_input_path, model_name)):
158
- results = combine_eval_results(base_bm_input_path, model_name, "base")
159
- # TMP: Add missing benchmarks to the results
160
- for metric in METRIC_NAME.items():
161
- if metric[0] not in results["results"]:
162
- results["results"].update({metric[0]: {metric[1]: None}})
163
-
164
- if os.path.isdir(os.path.join(agentic_bm_input_path, AGENTIC_LOG_MODEL_NAME_MAP.get(model_name, "NA"))):
165
- agentic_bm_results = combine_eval_results(agentic_bm_input_path, model_name, "agentic")
166
- results["results"].update(agentic_bm_results["results"])
167
- with open(os.path.join(EVAL_RESULTS_PATH, f"{model_name}.json"), "w") as f:
168
- json.dump(results, f, indent=4)
169
-
170
- # Create dummy requests file
171
- requests = {
172
- "model": model_name,
173
- "model_sha": MODEL_SHA_MAP[model_name],
174
- "model_version": MODEL_VERSION_MAP[model_name],
175
- "base_model": "",
176
- "revision": "main",
177
- "private": False,
178
- "precision": "float16",
179
- "weight_type": "Original",
180
- "status": "FINISHED",
181
- "submitted_time": "",
182
- "model_type": "pretrained",
183
- "likes": 0,
184
- "params": 0,
185
- "license": "custom",
186
- }
187
- with open(os.path.join(EVAL_REQUESTS_PATH, f"{model_name}.json"), "w") as f:
188
- json.dump(requests, f, indent=4)
189
-
190
-
191
- if __name__ == "__main__":
192
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/about.py CHANGED
@@ -1,50 +1,9 @@
1
- from dataclasses import dataclass
2
- from enum import Enum
3
-
4
- @dataclass
5
- class Task:
6
- benchmark: str
7
- metric: str
8
- col_name: str
9
- type: str
10
- source: str
11
-
12
-
13
- # Select your tasks here
14
- # ---------------------------------------------------
15
- class Tasks(Enum):
16
- # task_key in the json file, metric_key in the json file, name to display in the leaderboard
17
-
18
- # base
19
- task0 = Task("arc_easy", "accuracy", "ARC-Easy", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/arc")
20
- task1 = Task("arc_challenge", "accuracy", "ARC-Challenge", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/arc")
21
- task2 = Task("drop", "mean", "DROP", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/drop")
22
- task3 = Task("winogrande", "accuracy", "WinoGrande", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/winogrande")
23
- task4 = Task("gsm8k", "accuracy", "GSM8K", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gsm8k")
24
- task5 = Task("hellaswag", "accuracy", "HellaSwag", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/hellaswag")
25
- task6 = Task("humaneval", "mean", "HumanEval", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/humaneval")
26
- task7 = Task("ifeval", "final_acc", "IFEval", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/ifeval")
27
- task8 = Task("math", "accuracy", "MATH", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mathematics")
28
- task9 = Task("mmlu", "accuracy", "MMLU", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmlu")
29
- task10 = Task("mmlu_pro", "accuracy", "MMLU-Pro", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmlu_pro")
30
- task11 = Task("gpqa_diamond", "accuracy", "GPQA-Diamond", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gpqa")
31
- task12 = Task("mmmu_multiple_choice", "accuracy", "MMMU-Multiple-Choice", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmmu")
32
- task13 = Task("mmmu_open", "accuracy", "MMMU-Open-Ended", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmmu")
33
-
34
- # agentic
35
- task14 = Task("gaia", "accuracy", "GAIA", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gaia")
36
- task15 = Task("gdm_intercode_ctf", "accuracy", "InterCode-CTF", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/intercode_ctf")
37
- task16 = Task("gdm_in_house_ctf", "accuracy", "In-House-CTF", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/in_house_ctf")
38
- task17 = Task("agentharm", "avg_score", "AgentHarm", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentharm")
39
- task18 = Task("agentharm_benign", "avg_score", "AgentHarm-Benign", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentharm")
40
- task19 = Task("swe_bench", "mean", "SWE-Bench", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/swe_bench")
41
-
42
 
43
  # Your leaderboard name
44
  TITLE = """<h1 align="center" id="space-title">State of Evaluation Leaderboard</h1>"""
45
 
46
- SINGLE_TURN_TASK_NAMES = ", ".join([f"[{task.value.col_name}]({task.value.source})" for task in Tasks if task.value.type == "base"])
47
- AGENTIC_TASK_NAMES = ", ".join([f"[{task.value.col_name}]({task.value.source})" for task in Tasks if task.value.type == "agentic"])
48
 
49
  # What does your leaderboard evaluate?
50
  INTRODUCTION_TEXT = f"""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
 
2
  # Your leaderboard name
3
  TITLE = """<h1 align="center" id="space-title">State of Evaluation Leaderboard</h1>"""
4
 
5
+ # SINGLE_TURN_TASK_NAMES = ", ".join([f"[{task.value.col_name}]({task.value.source})" for task in Tasks if task.value.type == "base"])
6
+ # AGENTIC_TASK_NAMES = ", ".join([f"[{task.value.col_name}]({task.value.source})" for task in Tasks if task.value.type == "agentic"])
7
 
8
  # What does your leaderboard evaluate?
9
  INTRODUCTION_TEXT = f"""
src/display/utils.py DELETED
@@ -1,102 +0,0 @@
1
- from dataclasses import dataclass, make_dataclass
2
- from enum import Enum
3
-
4
- import pandas as pd
5
-
6
- from src.about import Tasks
7
-
8
- def fields(raw_class):
9
- return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
10
-
11
-
12
- # These classes are for user facing column names,
13
- # to avoid having to change them all around the code
14
- # when a modif is needed
15
- @dataclass
16
- class ColumnContent:
17
- name: str
18
- type: str
19
- displayed_by_default: bool
20
- hidden: bool = False
21
- never_hidden: bool = False
22
-
23
- def __hash__(self):
24
- return f"{self.name}-{self.type}-{self.displayed_by_default}-{self.hidden}-{self.never_hidden}"
25
-
26
- ## Leaderboard columns
27
- auto_eval_column_dict = []
28
- # Init
29
- auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
30
- auto_eval_column_dict.append(["agent", ColumnContent, ColumnContent("Agent", "markdown", True, never_hidden=True)])
31
- # Scores
32
- for task in Tasks:
33
- auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "markdown", True)])
34
-
35
- # We use make dataclass to dynamically fill the scores from Tasks
36
- AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
37
-
38
- ## For the queue columns in the submission tab
39
- @dataclass(frozen=True)
40
- class EvalQueueColumn: # Queue column
41
- model = ColumnContent("model", "markdown", True)
42
- revision = ColumnContent("revision", "str", True)
43
- private = ColumnContent("private", "bool", True)
44
- precision = ColumnContent("precision", "str", True)
45
- weight_type = ColumnContent("weight_type", "str", "Original")
46
- status = ColumnContent("status", "str", True)
47
-
48
- ## All the model information that we might need
49
- @dataclass
50
- class ModelDetails:
51
- name: str
52
- display_name: str = ""
53
- symbol: str = "" # emoji
54
-
55
-
56
- class ModelType(Enum):
57
- PT = ModelDetails(name="pretrained", symbol="🟢")
58
- FT = ModelDetails(name="fine-tuned", symbol="🔶")
59
- IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
60
- RL = ModelDetails(name="RL-tuned", symbol="🟦")
61
- Unknown = ModelDetails(name="", symbol="?")
62
-
63
- def to_str(self, separator=" "):
64
- return f"{self.value.symbol}{separator}{self.value.name}"
65
-
66
- @staticmethod
67
- def from_str(type):
68
- if "fine-tuned" in type or "🔶" in type:
69
- return ModelType.FT
70
- if "pretrained" in type or "🟢" in type:
71
- return ModelType.PT
72
- if "RL-tuned" in type or "🟦" in type:
73
- return ModelType.RL
74
- if "instruction-tuned" in type or "⭕" in type:
75
- return ModelType.IFT
76
- return ModelType.Unknown
77
-
78
- class WeightType(Enum):
79
- Adapter = ModelDetails("Adapter")
80
- Original = ModelDetails("Original")
81
- Delta = ModelDetails("Delta")
82
-
83
- class Precision(Enum):
84
- float16 = ModelDetails("float16")
85
- bfloat16 = ModelDetails("bfloat16")
86
- Unknown = ModelDetails("?")
87
-
88
- def from_str(precision):
89
- if precision in ["torch.float16", "float16"]:
90
- return Precision.float16
91
- if precision in ["torch.bfloat16", "bfloat16"]:
92
- return Precision.bfloat16
93
- return Precision.Unknown
94
-
95
- # Column selection
96
- COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
97
-
98
- EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
99
- EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
100
-
101
- ST_BENCHMARK_COLS = [t.value.col_name for t in Tasks if t.value.type=="base"]
102
- AGENTIC_BENCHMARK_COLS = [t.value.col_name for t in Tasks if t.value.type=="agentic"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/envs.py DELETED
@@ -1,25 +0,0 @@
1
- import os
2
-
3
- from huggingface_hub import HfApi
4
-
5
- # Info to change for your repository
6
- # ----------------------------------
7
- TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
8
-
9
- OWNER = "vector-institute" # Change to your org - don't forget to create a results and request dataset, with the correct format!
10
- # ----------------------------------
11
-
12
- REPO_ID = f"{OWNER}/llm-eval-leaderboard"
13
- QUEUE_REPO = f"{OWNER}/llm-eval-requests"
14
- RESULTS_REPO = f"{OWNER}/llm-eval-results"
15
-
16
- # If you setup a cache later, just change HF_HOME
17
- CACHE_PATH=os.getenv("HF_HOME", ".")
18
-
19
- # Local caches
20
- EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
21
- EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
22
- EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
23
- EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
24
-
25
- API = HfApi(token=TOKEN)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/leaderboard/read_evals.py DELETED
@@ -1,192 +0,0 @@
1
- import glob
2
- import json
3
- import math
4
- import os
5
- from dataclasses import dataclass
6
-
7
- import dateutil
8
- import numpy as np
9
-
10
- from src.display.formatting import make_clickable_model
11
- from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
12
- from src.submission.check_validity import is_model_on_hub
13
-
14
-
15
- @dataclass
16
- class EvalResult:
17
- """Represents one full evaluation. Built from a combination of the result and request file for a given run.
18
- """
19
- eval_name: str # org_model_precision (uid)
20
- full_model: str # org/model (path on hub)
21
- org: str
22
- model: str
23
- revision: str # commit hash, "" if main
24
- results: dict
25
- model_version: str = ""
26
- precision: Precision = Precision.Unknown
27
- model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
28
- weight_type: WeightType = WeightType.Original # Original or Adapter
29
- architecture: str = "Unknown"
30
- license: str = "?"
31
- likes: int = 0
32
- num_params: int = 0
33
- date: str = "" # submission date of request file
34
- still_on_hub: bool = False
35
-
36
- @classmethod
37
- def init_from_json_file(self, json_filepath):
38
- """Inits the result from the specific model result file"""
39
- with open(json_filepath) as fp:
40
- data = json.load(fp)
41
-
42
- config = data.get("config")
43
-
44
- # Precision
45
- precision = Precision.from_str(config.get("model_dtype"))
46
-
47
- # Get model and org
48
- org_and_model = config.get("model_name", config.get("model_args", None))
49
- org_and_model = org_and_model.split("/", 1)
50
-
51
- if len(org_and_model) == 1:
52
- org = None
53
- model = org_and_model[0]
54
- result_key = f"{model}"
55
- else:
56
- org = org_and_model[0]
57
- model = org_and_model[1]
58
- result_key = f"{org}_{model}"
59
- full_model = "/".join(org_and_model)
60
-
61
- still_on_hub, _, model_config = is_model_on_hub(
62
- full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
63
- )
64
- architecture = "?"
65
- if model_config is not None:
66
- architectures = getattr(model_config, "architectures", None)
67
- if architectures:
68
- architecture = ";".join(architectures)
69
-
70
- # Extract results available in this file (some results are split in several files)
71
- results = {}
72
- for task in Tasks:
73
- task = task.value
74
-
75
- # We average all scores of a given metric (not all metrics are present in all files)
76
- accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
77
- # if accs.size == 0 or any([acc is None for acc in accs]):
78
- # continue
79
- if accs.size == 0:
80
- continue
81
- elif any([acc is None for acc in accs]):
82
- mean_acc = None
83
- else:
84
- mean_acc = np.mean(accs) * 100.0
85
- results[task.benchmark] = mean_acc
86
-
87
- return self(
88
- eval_name=result_key,
89
- full_model=full_model,
90
- org=org,
91
- model=model,
92
- results=results,
93
- precision=precision,
94
- revision= config.get("model_sha", ""),
95
- still_on_hub=still_on_hub,
96
- architecture=architecture
97
- )
98
-
99
- def update_with_request_file(self, requests_path):
100
- """Finds the relevant request file for the current model and updates info with it"""
101
- request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
102
-
103
- try:
104
- with open(request_file, "r") as f:
105
- request = json.load(f)
106
- self.model_type = ModelType.from_str(request.get("model_type", ""))
107
- self.model_version = request.get("model_version", "")
108
- self.weight_type = WeightType[request.get("weight_type", "Original")]
109
- self.license = request.get("license", "?")
110
- self.likes = request.get("likes", 0)
111
- self.num_params = request.get("params", 0)
112
- self.date = request.get("submitted_time", "")
113
- except Exception:
114
- print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
115
-
116
- def to_dict(self):
117
- """Converts the Eval Result to a dict compatible with our dataframe display"""
118
- data_dict = {
119
- "eval_name": self.eval_name, # not a column, just a save name,
120
- AutoEvalColumn.model.name: make_clickable_model(self.model_version, self.revision),
121
- # As of now all models use the basic inspect agent
122
- AutoEvalColumn.agent.name: "[Basic Agent](https://inspect.ai-safety-institute.org.uk/agents.html#sec-basic-agent)"
123
- }
124
-
125
- for task in Tasks:
126
- data_dict[task.value.col_name] = self.results[task.value.benchmark]
127
-
128
- return data_dict
129
-
130
-
131
- def get_request_file_for_model(requests_path, model_name, precision):
132
- """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
133
- request_files = os.path.join(
134
- requests_path,
135
- f"{model_name}.json",
136
- )
137
- request_files = glob.glob(request_files)
138
-
139
- # Select correct request file (precision)
140
- request_file = ""
141
- request_files = sorted(request_files, reverse=True)
142
- for tmp_request_file in request_files:
143
- with open(tmp_request_file, "r") as f:
144
- req_content = json.load(f)
145
- if (
146
- req_content["status"] in ["FINISHED"]
147
- and req_content["precision"] == precision.split(".")[-1]
148
- ):
149
- request_file = tmp_request_file
150
- return request_file
151
-
152
-
153
- def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
154
- """From the path of the results folder root, extract all needed info for results"""
155
- model_result_filepaths = []
156
-
157
- for root, _, files in os.walk(results_path):
158
- # We should only have json files in model results
159
- if len(files) == 0 or any([not f.endswith(".json") for f in files]):
160
- continue
161
-
162
- # Sort the files by date
163
- try:
164
- files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
165
- except dateutil.parser._parser.ParserError:
166
- files = [files[-1]]
167
-
168
- for file in files:
169
- model_result_filepaths.append(os.path.join(root, file))
170
-
171
- eval_results = {}
172
- for model_result_filepath in model_result_filepaths:
173
- # Creation of result
174
- eval_result = EvalResult.init_from_json_file(model_result_filepath)
175
- eval_result.update_with_request_file(requests_path)
176
-
177
- # Store results of same eval together
178
- eval_name = eval_result.eval_name
179
- if eval_name in eval_results.keys():
180
- eval_results[eval_name].results.update(eval_result.results)
181
- else:
182
- eval_results[eval_name] = eval_result
183
-
184
- results = []
185
- for v in eval_results.values():
186
- try:
187
- v.to_dict() # we test if the dict version is complete
188
- results.append(v)
189
- except KeyError: # not all eval values present
190
- continue
191
-
192
- return results
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/populate.py DELETED
@@ -1,98 +0,0 @@
1
- import json
2
- import os
3
-
4
- import numpy as np
5
- import pandas as pd
6
-
7
- from src.about import Tasks
8
- from src.display.formatting import has_no_nan_values, make_clickable_model
9
- from src.display.utils import AutoEvalColumn, EvalQueueColumn
10
- from src.leaderboard.read_evals import get_raw_eval_results
11
-
12
- from refactor_eval_results import MODEL_VERSION_MAP
13
-
14
-
15
- TASK_NAME_INVERSE_MAP = dict()
16
- for task in Tasks:
17
- TASK_NAME_INVERSE_MAP[task.value.col_name] = {
18
- "name": task.value.benchmark,
19
- "type": task.value.type,
20
- "source": task.value.source,
21
- }
22
-
23
- EMPTY_SYMBOL = "--"
24
-
25
-
26
- def get_inspect_log_url(model_name: str, benchmark_name: str) -> str:
27
- """Returns the URL to the log file for a given model and benchmark"""
28
- with open("./inspect_log_file_names.json", "r") as f:
29
- inspect_log_files = json.load(f)
30
- log_file_name = inspect_log_files[model_name].get(benchmark_name, None)
31
- if log_file_name is None:
32
- return ""
33
- else:
34
- # replace .json with .eval
35
- log_file_name = log_file_name.replace(".json", ".eval")
36
- return f"https://storage.googleapis.com/inspect-evals/eval/{model_name}/index.html?log_file=logs/logs/{log_file_name}"
37
-
38
-
39
- def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
40
- """Creates a dataframe from all the individual experiment results"""
41
- raw_data = get_raw_eval_results(results_path, requests_path)
42
- all_data_json = [v.to_dict() for v in raw_data]
43
-
44
- df = pd.DataFrame.from_records(all_data_json)
45
-
46
- df = df[cols].round(decimals=2)
47
-
48
- # subset for model and benchmark cols
49
- df = df[[AutoEvalColumn.model.name, AutoEvalColumn.agent.name] + benchmark_cols]
50
-
51
- # drop rows for which all benchmark cols are empty
52
- df = df.dropna(subset=benchmark_cols, axis=0, how="all")
53
-
54
- df = df.fillna(EMPTY_SYMBOL)
55
-
56
- inverse_model_version_map = {v: k for k, v in MODEL_VERSION_MAP.items()}
57
-
58
- # make values clickable and link to log files
59
- for col in benchmark_cols:
60
- df[col] = df[[AutoEvalColumn.model.name, col]].apply(lambda x: f"[{x[col]}]({get_inspect_log_url(model_name=inverse_model_version_map[x[AutoEvalColumn.model.name].split('>')[1].split('<')[0]], benchmark_name=TASK_NAME_INVERSE_MAP[col]['name'])})" if x[col] != EMPTY_SYMBOL else x[col], axis=1)
61
-
62
- return df
63
-
64
-
65
- def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
66
- """Creates the different dataframes for the evaluation queues requestes"""
67
- entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
68
- all_evals = []
69
-
70
- for entry in entries:
71
- if ".json" in entry:
72
- file_path = os.path.join(save_path, entry)
73
- with open(file_path) as fp:
74
- data = json.load(fp)
75
-
76
- data[EvalQueueColumn.model.name] = make_clickable_model(data["model"], data["model_sha"])
77
- data[EvalQueueColumn.revision.name] = data.get("revision", "main")
78
-
79
- all_evals.append(data)
80
- elif ".md" not in entry:
81
- # this is a folder
82
- sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
83
- for sub_entry in sub_entries:
84
- file_path = os.path.join(save_path, entry, sub_entry)
85
- with open(file_path) as fp:
86
- data = json.load(fp)
87
-
88
- data[EvalQueueColumn.model.name] = make_clickable_model(data["model"], data["model_sha"])
89
- data[EvalQueueColumn.revision.name] = data.get("revision", "main")
90
- all_evals.append(data)
91
-
92
- pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
93
- running_list = [e for e in all_evals if e["status"] == "RUNNING"]
94
- finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
95
- df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
96
- df_running = pd.DataFrame.from_records(running_list, columns=cols)
97
- df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
98
- return df_finished[cols], df_running[cols], df_pending[cols]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/submission/check_validity.py DELETED
@@ -1,99 +0,0 @@
1
- import json
2
- import os
3
- import re
4
- from collections import defaultdict
5
- from datetime import datetime, timedelta, timezone
6
-
7
- import huggingface_hub
8
- from huggingface_hub import ModelCard
9
- from huggingface_hub.hf_api import ModelInfo
10
- from transformers import AutoConfig
11
- from transformers.models.auto.tokenization_auto import AutoTokenizer
12
-
13
- def check_model_card(repo_id: str) -> tuple[bool, str]:
14
- """Checks if the model card and license exist and have been filled"""
15
- try:
16
- card = ModelCard.load(repo_id)
17
- except huggingface_hub.utils.EntryNotFoundError:
18
- return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
19
-
20
- # Enforce license metadata
21
- if card.data.license is None:
22
- if not ("license_name" in card.data and "license_link" in card.data):
23
- return False, (
24
- "License not found. Please add a license to your model card using the `license` metadata or a"
25
- " `license_name`/`license_link` pair."
26
- )
27
-
28
- # Enforce card content
29
- if len(card.text) < 200:
30
- return False, "Please add a description to your model card, it is too short."
31
-
32
- return True, ""
33
-
34
- def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
35
- """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
36
- try:
37
- config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
38
- if test_tokenizer:
39
- try:
40
- tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
41
- except ValueError as e:
42
- return (
43
- False,
44
- f"uses a tokenizer which is not in a transformers release: {e}",
45
- None
46
- )
47
- except Exception as e:
48
- return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
49
- return True, None, config
50
-
51
- except ValueError:
52
- return (
53
- False,
54
- "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
55
- None
56
- )
57
-
58
- except Exception as e:
59
- return False, "was not found on hub!", None
60
-
61
-
62
- def get_model_size(model_info: ModelInfo, precision: str):
63
- """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
64
- try:
65
- model_size = round(model_info.safetensors["total"] / 1e9, 3)
66
- except (AttributeError, TypeError):
67
- return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
68
-
69
- size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
70
- model_size = size_factor * model_size
71
- return model_size
72
-
73
- def get_model_arch(model_info: ModelInfo):
74
- """Gets the model architecture from the configuration"""
75
- return model_info.config.get("architectures", "Unknown")
76
-
77
- def already_submitted_models(requested_models_dir: str) -> set[str]:
78
- """Gather a list of already submitted models to avoid duplicates"""
79
- depth = 1
80
- file_names = []
81
- users_to_submission_dates = defaultdict(list)
82
-
83
- for root, _, files in os.walk(requested_models_dir):
84
- current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
85
- if current_depth == depth:
86
- for file in files:
87
- if not file.endswith(".json"):
88
- continue
89
- with open(os.path.join(root, file), "r") as f:
90
- info = json.load(f)
91
- file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
92
-
93
- # Select organisation
94
- if info["model"].count("/") == 0 or "submitted_time" not in info:
95
- continue
96
- organisation, _ = info["model"].split("/")
97
- users_to_submission_dates[organisation].append(info["submitted_time"])
98
-
99
- return set(file_names), users_to_submission_dates
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/submission/submit.py DELETED
@@ -1,119 +0,0 @@
1
- # import json
2
- # import os
3
- # from datetime import datetime, timezone
4
-
5
- # from src.display.formatting import styled_error, styled_message, styled_warning
6
- # from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
7
- # from src.submission.check_validity import (
8
- # already_submitted_models,
9
- # check_model_card,
10
- # get_model_size,
11
- # is_model_on_hub,
12
- # )
13
-
14
- # REQUESTED_MODELS = None
15
- # USERS_TO_SUBMISSION_DATES = None
16
-
17
- # def add_new_eval(
18
- # model: str,
19
- # base_model: str,
20
- # revision: str,
21
- # precision: str,
22
- # weight_type: str,
23
- # model_type: str,
24
- # ):
25
- # global REQUESTED_MODELS
26
- # global USERS_TO_SUBMISSION_DATES
27
- # if not REQUESTED_MODELS:
28
- # REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
29
-
30
- # user_name = ""
31
- # model_path = model
32
- # if "/" in model:
33
- # user_name = model.split("/")[0]
34
- # model_path = model.split("/")[1]
35
-
36
- # precision = precision.split(" ")[0]
37
- # current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
38
-
39
- # if model_type is None or model_type == "":
40
- # return styled_error("Please select a model type.")
41
-
42
- # # Does the model actually exist?
43
- # if revision == "":
44
- # revision = "main"
45
-
46
- # # Is the model on the hub?
47
- # if weight_type in ["Delta", "Adapter"]:
48
- # base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
49
- # if not base_model_on_hub:
50
- # return styled_error(f'Base model "{base_model}" {error}')
51
-
52
- # if not weight_type == "Adapter":
53
- # model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
54
- # if not model_on_hub:
55
- # return styled_error(f'Model "{model}" {error}')
56
-
57
- # # Is the model info correctly filled?
58
- # try:
59
- # model_info = API.model_info(repo_id=model, revision=revision)
60
- # except Exception:
61
- # return styled_error("Could not get your model information. Please fill it up properly.")
62
-
63
- # model_size = get_model_size(model_info=model_info, precision=precision)
64
-
65
- # # Were the model card and license filled?
66
- # try:
67
- # license = model_info.cardData["license"]
68
- # except Exception:
69
- # return styled_error("Please select a license for your model")
70
-
71
- # modelcard_OK, error_msg = check_model_card(model)
72
- # if not modelcard_OK:
73
- # return styled_error(error_msg)
74
-
75
- # # Seems good, creating the eval
76
- # print("Adding new eval")
77
-
78
- # eval_entry = {
79
- # "model": model,
80
- # "base_model": base_model,
81
- # "revision": revision,
82
- # "precision": precision,
83
- # "weight_type": weight_type,
84
- # "status": "PENDING",
85
- # "submitted_time": current_time,
86
- # "model_type": model_type,
87
- # "likes": model_info.likes,
88
- # "params": model_size,
89
- # "license": license,
90
- # "private": False,
91
- # }
92
-
93
- # # Check for duplicate submission
94
- # if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
95
- # return styled_warning("This model has been already submitted.")
96
-
97
- # print("Creating eval file")
98
- # OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
99
- # os.makedirs(OUT_DIR, exist_ok=True)
100
- # out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
101
-
102
- # with open(out_path, "w") as f:
103
- # f.write(json.dumps(eval_entry))
104
-
105
- # print("Uploading eval file")
106
- # API.upload_file(
107
- # path_or_fileobj=out_path,
108
- # path_in_repo=out_path.split("eval-queue/")[1],
109
- # repo_id=QUEUE_REPO,
110
- # repo_type="dataset",
111
- # commit_message=f"Add {model} to eval queue",
112
- # )
113
-
114
- # # Remove the local file
115
- # os.remove(out_path)
116
-
117
- # return styled_message(
118
- # "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
119
- # )