Add agentharm and swe-bench tasks

inspect_log_file_names.json

@@ -16,7 +16,10 @@
         "hellaswag": "2024-11-05T13-14-31-05-00_hellaswag_N98eeftuY2pucRtgpUYk5m.json",
         "gaia": "2025-01-21T15-33-29-05-00_gemini-1.5-pro_gaia_merged.json",
         "gdm_intercode_ctf": "2025-01-21T23-59-58+00-00_gemini-1.5-pro_gdm-intercode-ctf_merged.json",
-        "gdm_in_house_ctf": "2025-01-22T03-42-16+00-00_gemini-1.5-pro_gdm-in-house-ctf.json"
+        "gdm_in_house_ctf": "2025-01-22T03-42-16+00-00_gemini-1.5-pro_gdm-in-house-ctf.json",
+        "agentharm_benign": "2025-01-21T13-18-51-08-00_agentharm-benign_gP3pQPxAuCtFLiHzt2Egt7.json",
+        "agentharm": "2025-01-21T12-45-43-08-00_agentharm_VmD26soLwmRgWPo3hpRHBr.json",
+        "swe_bench": "2025-01-22T03-00-08+00-00_google-gemini-1.5-pro.json"
     },
     "gemini-1.5-flash": {
         "gpqa_diamond": "2024-11-04T12-47-34-05-00_gpqa-diamond_cL5kQj8DWbRfxz79piTSdy.json",
@@ -47,7 +50,10 @@
         "mmmu_multiple_choice": "2025-01-20T21-04-57-05-00_mmmu-multiple-choice_MctxjookaeTLCL8KpUeazT.json",
         "gaia": "2025-01-22T13-42-00-05-00_o1_gaia_merged.json",
         "gdm_intercode_ctf": "2025-01-22T20-46-35+00-00_o1_gdm-intercode-ctf_merged.json",
-        "gdm_in_house_ctf": "2025-01-22T05-52-25+00-00_o1_gdm-in-house-ctf.json"
+        "gdm_in_house_ctf": "2025-01-22T05-52-25+00-00_o1_gdm-in-house-ctf.json",
+        "agentharm": "2025-01-21T09-05-42-08-00_agentharm_UGDq2yJeLAnPH6p7FgDgD8.json",
+        "agentharm_benign": "2025-01-21T18-20-15-08-00_agentharm-benign_bkW2Bf5xLyDQdNtfLdjCpJ.json",
+        "swe_bench": "2025-01-21T17-42-11+00-00_openai-o1.json"
     },
     "claude-3-5-sonnet-20241022": {
         "mmmu_multiple_choice": "2025-01-21T11-20-03-05-00_mmmu-multiple-choice_CWhKvGdoFo6pdHhDyi9GNm.json",
@@ -66,7 +72,10 @@
         "drop": "2025-01-15T10-15-15-05-00_drop_Z9A2Y84HYponNxnzNT9TNq.json",
         "gaia": "2025-01-12T23-57-37-05-00_claude-3-5-sonnet_gaia_merged.json",
         "gdm_intercode_ctf": "2025-01-11T02-47-45-05-00_claude-3-5-sonnet_gdm-intercode-ctf_merged.json",
-        "gdm_in_house_ctf": "2025-01-11T07-41-14+00-00_claude-3-5-sonnet_gdm-in-house-ctf.json"
+        "gdm_in_house_ctf": "2025-01-11T07-41-14+00-00_claude-3-5-sonnet_gdm-in-house-ctf.json",
+        "agentharm_benign": "2025-01-21T15-09-48-08-00_agentharm-benign_A3uBBWNvv88P5BsgqwFCfg.json",
+        "agentharm": "2025-01-15T08-05-14-08-00_agentharm_VJGhWKLrVLdQczBZVgCXHc.json",
+        "swe_bench": "2025-01-16T18-56-55+00-00_anthropic-claude-3-5-sonnet.json"
     },
     "c4ai-command-r-plus": {
         "ifeval": "2024-10-30T17-23-04-04-00_ifeval_RGucUMwdGmUnRpqyMTZTzW.json",
@@ -129,7 +138,10 @@
         "hellaswag": "2024-10-31T03-33-47-04-00_hellaswag_JNnnPuz3dhZRpyXzizMUBF.json",
         "gaia": "2025-01-13T15-53-22+00-00_gpt-4o_gaia_merged.json",
         "gdm_intercode_ctf": "2025-01-08T10-06-29-05-00_gpt-4o_gdm-intercode-ctf_merged.json",
-        "gdm_in_house_ctf": "2025-01-11T07-02-14+00-00_gpt-4o_gdm-in-house-ctf.json"
+        "gdm_in_house_ctf": "2025-01-11T07-02-14+00-00_gpt-4o_gdm-in-house-ctf.json",
+        "agentharm": "2025-01-07T16-34-15-08-00_agentharm_UfSoyHEAH2E5RVdrPVUemy.json",
+        "agentharm_benign": "2025-01-21T13-45-18-08-00_agentharm-benign_8DhGJqEAvw6o8uCv4a4dVz.json",
+        "swe_bench": "2025-01-14T23-09-10+00-00_openai-gpt-4o.json"
     },
     "Mistral-Large-Instruct-2407": {
         "drop": "2024-10-31T01-56-12-04-00_drop_NtvuCoU2LoMbH8DztcCTen.json",

refactor_eval_results.py

@@ -23,6 +23,9 @@ METRIC_NAME = {
     "gaia": "accuracy",
     "gdm_intercode_ctf": "accuracy",
     "gdm_in_house_ctf": "accuracy",
+    "agentharm": "avg_score",
+    "agentharm_benign": "avg_score",
+    "swe_bench": "mean",
 }
 
 MODEL_SHA_MAP = {
@@ -48,7 +51,7 @@ AGENTIC_LOG_MODEL_NAME_MAP = {
     "o1": "o1-2024-12-17",
 }
 
-AGENTIC_TASKS = ["gaia", "gdm-intercode-ctf", "gdm-in-house-ctf"]
+AGENTIC_TASKS = ["gaia", "gdm-intercode-ctf", "gdm-in-house-ctf", "agentharm", "swe-bench"]
 
 
 def combine_eval_results(results_path: str, model_name: str, type: str,) -> dict:
@@ -100,7 +103,7 @@ def combine_eval_results(results_path: str, model_name: str, type: str,) -> dict
                             result = json.load(f)
                             task_name = result["eval"]["task"].split("/")[-1]
                             metrics = result["results"]["scores"][0]["metrics"]
-                            metric_name = metrics[METRIC_NAME[task_name]]["name"]
+                            metric_name = metrics[METRIC_NAME[task_name]]["name"].split("/")[-1]
                             metric_value = metrics[METRIC_NAME[task_name]]["value"]
                             results["results"].update(
                                 {

src/about.py

@@ -35,6 +35,9 @@ class Tasks(Enum):
     task14 = Task("gaia", "accuracy", "GAIA", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gaia")
     task15 = Task("gdm_intercode_ctf", "accuracy", "GDM-InterCode-CTF", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/intercode_ctf")
     task16 = Task("gdm_in_house_ctf", "accuracy", "GDM-In-House-CTF", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/in_house_ctf")
+    task17 = Task("agentharm", "avg_score", "AgentHarm", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentharm")
+    task18 = Task("agentharm_benign", "avg_score", "AgentHarm-Benign", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentharm")
+    task19 = Task("swe_bench", "mean", "SWE-Bench", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/swe_bench")
 
 
 NUM_FEWSHOT = 0 # Change with your few shot
@@ -60,7 +63,7 @@ LLM_BENCHMARKS_TEXT = f"""
 The **Vector State of Evaluation Leaderboard** presents the performance of selected LLM models on a variety of tasks. These tasks are divided into two categories:
 
 - **Base Tasks**: ARC-Easy, ARC-Challenge, DROP, WinoGrande, GSM8K, HellaSwag, HumanEval, IFEval, MATH, MMLU, MMLU-Pro, GPQA-Diamond, MMMU-Multiple-Choice, MMMU-Open-Ended  
-- **Agentic Tasks**: GAIA, GDM-InterCode-CTF, GDM-In-House-CTF
+- **Agentic Tasks**: GAIA, GDM-InterCode-CTF, GDM-In-House-CTF, AgentHarm, AgentHarm-Benign, SWE-Bench
 
 Users can compare models side by side to see how they perform on both base-level understanding tasks and more advanced, “agentic” tasks.
 
@@ -95,6 +98,8 @@ Here is a closer look at each benchmark included in the leaderboard:
 - **GAIA**: Evaluates more autonomous or “agentic” reasoning, including planning and problem-solving.
 - **GDM-InterCode-CTF**: A capture-the-flag style challenge focusing on code interpretation and generative debugging strategies.
 - **GDM-In-House-CTF**: A capture-the-flag style challenge testing a variety of security skills pertaining to basic web application security.
+- **AgentHarm / AgentHarm-Benign**: A benchmark for measuring harmfulness of LLM agents.
+- **SWE-Bench**: A benchmark for testing the ability of AI agents to solve software engineering tasks.
 
 ---
 """