revising app

Signed-off-by: Yotam-Perlitz <[email protected]>

app.py

@@ -6,59 +6,226 @@ import plotly.express as px
 import streamlit as st
 from bat import Benchmark, Config, Reporter, Tester
 
-holistic_scenarios = [
-    "arena_hard",
-    "mixeval",
-    "agieval",
-    "arc_c",
-    "alpacav1",
-    "alpacav2",
-    "alpacaeval2_lc",
-    "arena_elo",
-    "bbh",
-    "eq_benchv2",
-    "gpt4all",
-    "hugging_6",
-    "llmonitor",
-    "magi",
-    "mmlu",
-    "mt_bench",
-    "biggen_mwr",
-    "olmes_average",
-    "mmlu_pro",
-]
-
 
 def get_nice_benchmark_name(bench_name):
-    benchmarks_dict = {
-        "arena_elo": "LMSys Arena",
-        "mt_bench": "MT Bench",
-        "mixeval": "Mix Eval",
-        "alpacav2": "AlpacaEval V2",
+    prettified_names = {
+        "holmes": "Holmes",
+        "helm_lite_narrativeqa": "Helm Lite NarrativeQA",
+        "helm_lite_naturalquestionsopen": "Helm Lite NaturalQuestionsOpen",
+        "helm_lite_naturalquestionsclosed": "Helm Lite NaturalQuestionsClosed",
+        "helm_lite_openbookqa": "Helm Lite OpenBookQA",
+        "helm_lite_mmlu": "Helm Lite MMLU",
+        "helm_lite_math_equivalentcot": "Helm Lite MathEquivalentCOT",
+        "helm_lite_gsm8k": "Helm Lite GSM8K",
+        "helm_lite_legalbench": "Helm Lite LegalBench",
+        "helm_lite_medqa": "Helm Lite MedQA",
+        "helm_lite_wmt2014": "Helm Lite WMT2014",
+        "hfv2_bbh": "HFv2 BBH",
+        "hfv2_bbh_raw": "HFv2 BBH Raw",
+        "hfv2_gpqa": "HFv2 GPQA",
+        "hfv2_ifeval": "HFv2 IFEval",
+        "hfv2_math_lvl_5": "HFv2 Math Level 5",
+        "hfv2_mmlu_pro": "HFv2 MMLU Pro",
+        "hfv2_musr": "HFv2 MuSR",
+        "oc_mmlu": "OpenCompass MMLU",
+        "oc_mmlu_pro": "OpenCompass MMLU Pro",
+        "oc_cmmlu": "OpenCompass CMMLU",
+        "oc_bbh": "OpenCompass BBH",
+        "oc_gqpa_dimand": "OpenCompass GQPA-Dimand",
+        "oc_humaneval": "OpenCompass HumanEval",
+        "oc_ifeval": "OpenCompass IFEval",
+        "helm_mmlu": "Helm MMLU",
+        "helm_boolq": "Helm BoolQ",
+        "helm_narrativeqa": "Helm NarrativeQA",
+        "helm_naturalquestionsclosed": "Helm NaturalQuestionsClosed",
+        "helm_naturalquestionsopen": "Helm NaturalQuestionsOpen",
+        "helm_quac": "Helm QuAC",
+        "helm_openbookqa": "Helm OpenBookQA",
+        "helm_imdb": "Helm IMDB",
+        "helm_civilcomments": "Helm CivilComments",
+        "helm_raft": "Helm RAFT",
+        "mmlu_pro": "MMLU Pro",
+        "mixeval_triviaqa": "MixEval TriviaQA",
+        "mixeval_mmlu": "MixEval MMLU",
+        "mixeval_drop": "MixEval DROP",
+        "mixeval_hellaswag": "MixEval HellaSwag",
+        "mixeval_commonsenseqa": "MixEval CommonsenseQA",
+        "mixeval_triviaqa_hard": "MixEval TriviaQA Hard",
+        "mixeval_mmlu_hard": "MixEval MMLU Hard",
+        "mixeval_drop_hard": "MixEval DROP Hard",
+        "oc_language": "OpenCompass Language",
+        "oc_knowledge": "OpenCompass Knowledge",
+        "oc_reasoning": "OpenCompass Reasoning",
+        "oc_math": "OpenCompass Math",
+        "oc_code": "OpenCompass Code",
+        "oc_instruct": "OpenCompass Instruction",
+        "oc_agent": "OpenCompass Agent",
+        "oc_arena": "OpenCompass Arena",
+        "lb_reasoning": "LiveBench Reasoning",
+        "lb_coding": "LiveBench Coding",
+        "lb_mathematics": "LiveBench Mathematics",
+        "lb_data_analysis": "LiveBench Data Analysis",
+        "lb_language": "LiveBench Language",
+        "lb_if": "LiveBench Instruction Following",
+        "wb_info_seek": "WildBench Information Seeking",
+        "wb_creative": "WildBench Creative",
+        "wb_code_debug": "WildBench Code Debugging",
+        "wb_math_data": "WildBench Math & Data",
+        "wb_reason_plan": "WildBench Reasoning & Planning",
+        "wb_score": "WildBench Score",
+        "hfv1_arc": "HFv1 ARC",
+        "hfv1_gsm8k": "HFv1 GSM8K",
+        "hfv1_hellaswag": "HFv1 HellaSwag",
+        "hfv1_mmlu": "HFv1 MMLU",
+        "hfv1_truthfulqa": "HFv1 TruthfulQA",
+        "hfv1_winogrande": "HFv1 Winogrande",
+        "biggen_grounding": "BigBench Grounding",
+        "biggen_instruction_following": "BigBench Instruction Following",
+        "biggen_planning": "BigBench Planning",
+        "biggen_reasoning": "BigBench Reasoning",
+        "biggen_refinement": "BigBench Refinement",
+        "biggen_safety": "BigBench Safety",
+        "biggen_theory_of_mind": "BigBench Theory of Mind",
+        "biggen_tool_usage": "BigBench Tool Usage",
+        "biggen_multilingual": "BigBench Multilingual",
+        "lb_reasoning_average": "LiveBench Reasoning Average",
+        "lb_coding_average": "LiveBench Coding Average",
+        "lb_mathematics_average": "LiveBench Mathematics Average",
+        "lb_data_analysis_average": "LiveBench Data Analysis Average",
+        "lb_language_average": "LiveBench Language Average",
+        "lb_if_average": "LiveBench Instruction Following Average",
+        "helm_lite": "Helm Lite",
+        "hf_open_llm_v2": "HF OpenLLM v2",
+        "opencompass_academic": "OpenCompass Academic",
+        "arena_elo": "Arena Elo",
+        "helm_classic": "Helm Classic",
+        "mixeval": "MixEval",
+        "mixeval_hard": "MixEval Hard",
+        "opencompass": "OpenCompass",
+        "alphacaeval_v2lc": "AlphacaEval v2lc",
+        "livebench_240725": "LiveBench 240725",
+        "wb_elo_lc": "WildBench Elo LC",
         "arena_hard": "Arena Hard",
-        "arc_c": "ARC-C",
-        "eq_benchv2": "EQ Bench V2",
-        "agieval": "AGIEval",
-        "llmonitor": "LLMonitor",
-        "bbh": "BBH",
-        "mmlu": "MMLU",
-        "alpacav1": "AlpacaEval V1",
-        "magi": "MAGI",
-        "alpacaeval2_lc": "AlpacaEval V2 Length Adjusted",
-        "gpt4all": "GPT-4-All",
-        "humaneval": "HumanEval",
-        "mbpp": "MBPP",
-        "hellaswag": "HellaSwag",
-        "hugging_6": "HF OpenLLM V1",
-        "winogrande": "Winogrande",
+        "agentbench": "AgentBench",
+        "hf_open_llm_v1": "HF OpenLLM v1",
+        "biggen": "BigBench",
+        "livebench_240624": "LiveBench 240624",
+        "mt_bench": "MT-Bench",
     }
 
-    if bench_name in benchmarks_dict:
-        return benchmarks_dict[bench_name]
+    if bench_name in prettified_names:
+        return prettified_names[bench_name]
     else:
         return bench_name
 
 
+holistic_scenarios = [
+    get_nice_benchmark_name(scen)
+    for scen in [
+        # "holmes",
+        "helm_lite",
+        # "narrativeqa",
+        # "naturalquestionsopen",
+        # "naturalquestionsclosed",
+        # "openbookqa",
+        # "mmlu",
+        # "math_equivalentcot",
+        # "gsm8k",
+        # "legalbench",
+        # "medqa",
+        # "wmt2014",
+        # "arc_c",
+        # "arc_e",
+        # "boolq",
+        # "csqa",
+        # "hellaswag",
+        # "piqa",
+        # "siqa",
+        # "winogrande",
+        # "olmes_average",
+        # "bbh",
+        # "bbh_raw",
+        # "gpqa",
+        "hf_open_llm_v2",
+        # "ifeval",
+        # "math_lvl_5",
+        # "mmlu_pro",
+        # "musr",
+        "opencompass_academic",
+        # "oc_mmlu",
+        # "oc_mmlu_pro",
+        # "oc_cmmlu",
+        # "oc_bbh",
+        # "oc_gqpa_dimand",
+        # "oc_math",
+        # "oc_humaneval",
+        # "oc_ifeval",
+        # "helm_mmlu",
+        "arena_elo",
+        "helm_classic",
+        # "quac",
+        # "truthfulqa",
+        # "ms_marcoregular",
+        # "ms_marcotrec",
+        # "cnn/dailymail",
+        # "xsum",
+        # "imdb",
+        # "civilcomments",
+        # "raft",
+        "mixeval_hard",
+        "mixeval",
+        # "arena_elo0527",
+        "opencompass",
+        # "oc_language",
+        # "oc_knowledge",
+        # "oc_reasoning",
+        # "oc_code",
+        # "oc_instruct",
+        # "oc_agent",
+        # "oc_arena",
+        "alphacaeval_v2lc",
+        "livebench_240725",
+        "livebench_240624",
+        # "lb_reasoning",
+        # "lb_coding",
+        # "lb_mathematics",
+        # "lb_data_analysis",
+        # "lb_language",
+        # "lb_if",
+        "wb_elo_lc",
+        # "wb_info_seek",
+        # "wb_creative",
+        # "wb_code_debug",
+        # "wb_math_data",
+        # "wb_reason_plan",
+        # "wb_score",
+        # "boolqmixed",
+        "arena_hard",
+        "agentbench",
+        # "arc",
+        "hf_open_llm_v1",
+        "biggen",
+        # "biggen_grounding",
+        # "biggen_instruction_following",
+        # "biggen_planning",
+        # "biggen_reasoning",
+        # "biggen_refinement",
+        # "biggen_safety",
+        # "biggen_theory_of_mind",
+        # "biggen_tool_usage",
+        # "biggen_multilingual",
+        # "lb_global_average",
+        # "lb_reasoning_average",
+        # "lb_coding_average",
+        # "lb_mathematics_average",
+        # "lb_data_analysis_average",
+        # "lb_language_average",
+        # "lb_if_average",
+        # "mt_bench",
+    ]
+]
+
+
 st.markdown(
     """<h1 style='text-align: center; color: black;'>🏋️‍♂️ BenchBench Leaderboard 🏋️‍♂️</h1>""",
     unsafe_allow_html=True,
@@ -70,7 +237,11 @@ st.markdown(
 )
 
 
-all_scenarios_for_aggragate = holistic_scenarios
+all_scenarios_for_aggragate = Benchmark()
+all_scenarios_for_aggragate.load_local_catalog()
+all_scenarios_for_aggragate = (
+    all_scenarios_for_aggragate.df["scenario"].unique().tolist()
+)
 
 st.subheader("The Leaderboard", divider=True)
 # st.subheader("🏋️‍♂️ BenchBench Leaderboard 🏋", divider=True)
@@ -79,29 +250,26 @@ leftcol, rightcol = st.columns([2, 1])
 
 with st.expander("Leaderboard configurations (defaults are great BTW)", icon="⚙️"):
     with st.form("my_form"):
-        all_scenarios_for_aggragate_with_all = all_scenarios_for_aggragate
-        all_scenarios_for_aggragate_with_all.append("All Holistic")
+        all_scenarios_for_aggragate_with_all = [
+            get_nice_benchmark_name(scenario)
+            for scenario in all_scenarios_for_aggragate
+        ]
 
         aggragate_scenarios = st.multiselect(
-            "Scenarios in Aggregate",
-            all_scenarios_for_aggragate_with_all,
-            ["All Holistic"],
-            # all_scenarios_for_aggragate,
+            "Scenarios in Aggregate (defualts are the 'Holistic' benchmarks)",
+            all_scenarios_for_aggragate,
+            holistic_scenarios,
         )
 
         corr_type = st.selectbox(
             label="Select Correlation type", options=["kendall", "pearson"], index=0
         )
 
-        aggragate_scenario_blacklist = (
-            [
-                scen
-                for scen in all_scenarios_for_aggragate
-                if scen not in aggragate_scenarios
-            ]
-            if "All Holistic" not in aggragate_scenarios
-            else []
-        )
+        aggragate_scenario_blacklist = [
+            scen
+            for scen in all_scenarios_for_aggragate
+            if scen not in aggragate_scenarios
+        ]
 
         model_select_strategy = st.selectbox(
             label="Select strategy",
@@ -109,7 +277,15 @@ with st.expander("Leaderboard configurations (defaults are great BTW)", icon="
             index=0,
         )
 
-        n_models_taken_list = [5]
+        n_models_taken_list = st.slider(
+            label="Select number of models to use",
+            min_value=3,
+            max_value=20,
+            value=10,
+        )
+
+        n_models_taken_list = [n_models_taken_list]
+
         n_exps = 10
 
         submitted = st.form_submit_button(label="Run BAT")
@@ -197,27 +373,18 @@ def run_load(
         # allbench.df = allbench.df[~allbench.df["source"].str.contains("livebench")]
 
         allbench.extend(my_benchmark)
-        allbench.df = allbench.df.drop(columns=["tag"])
+        # allbench.df = allbench.df.drop(columns=["tag"])
         allbench.clear_repeated_scenarios()
-        allbench.df = allbench.df.query("scenario not in @holistic_scenarios")
-
-        # allbench.df = allbench.df[~allbench.df["scenario"].str.contains("_mixed")]
-        # allbench.df = allbench.df[~allbench.df["scenario"].str.contains("agentbench")]
-
-        # st.dataframe(holistic.df.query('scenario=="aggregate"'))
 
+        # removing and adding the holistic scenarios
+        allbench.df = allbench.df.query("scenario not in @holistic_scenarios")
         allbench = allbench.extend(holistic)
 
         tester = Tester(cfg=cfg)
 
-        # len(allbench.get_scenario_appearences_count().keys())
-
-        allbench.df.query('source=="BlueBench"').model.unique()
-
-        allbench.df.query('scenario=="aggregate"').model.unique()
-
         agreements = tester.all_vs_all_agreement_testing(
-            allbench, single_source_scenario="aggregate"
+            allbench,
+            single_source_scenario="aggregate",  # olny measuring all with the aggragate
         )
 
         agreements.to_csv(cache_path, index=False)
@@ -236,12 +403,20 @@ agreements = run_load(
 
 reporter = Reporter()
 z_scores = reporter.get_all_z_scores(agreements=agreements, aggragate_name="aggregate")
+z_scores.drop(columns=["n_models_of_corr_with_agg"], inplace=True)
 
 corr_name = f"{'Kendall Tau' if corr_type=='kendall' else 'Per.'} Corr."
 
 z_scores["z_score"] = z_scores["z_score"].round(2)
 z_scores["corr_with_agg"] = z_scores["corr_with_agg"].round(2)
 z_scores["p_value_of_corr_with_agg"] = z_scores["p_value_of_corr_with_agg"].round(2)
+# z_scores["n_models_of_corr_with_agg"] = z_scores["n_models_of_corr_with_agg"].round(1)
+
+z_scores["source"] = z_scores["source"].apply(lambda x: x.split(".csv")[0])
+
+# print(z_scores["scenario"].unique().tolist())
+
+z_scores["scenario"] = z_scores["scenario"].apply(lambda x: get_nice_benchmark_name(x))
 
 data = (
     z_scores.rename(
@@ -249,7 +424,8 @@ data = (
             "scenario": "Benchmark",
             "z_score": "Z Score",
             "corr_with_agg": corr_name,
-            "p_value_of_corr_with_agg": "p value of Corr.",
+            "p_value_of_corr_with_agg": "p-value of Corr.",
+            # "n_models_of_corr_with_agg": "# Models Used",
             "source": "Source",
         }
     )
@@ -258,12 +434,6 @@ data = (
 )
 
 
-data = data[~data["Source"].str.contains("livebench")]
-data = data[~data["Source"].str.contains("biggen")]
-# data.drop(columns=["Source"], inplace=True)
-data["Benchmark"] = data["Benchmark"].apply(lambda x: get_nice_benchmark_name(x))
-
-
 # Apply coloring based on 'Z' valuesz
 def highlight_uploaded_benchmark(row):
     if row["Source"] == "Uploaded Benchmark":
@@ -279,16 +449,23 @@ styled_data = (
         vmin=-data["Z Score"].abs().max(),
         vmax=data["Z Score"].abs().max(),
     )
-    .format(subset=["Z Score", corr_name, "p value of Corr."], formatter="{:.2}")
     .apply(highlight_uploaded_benchmark, axis=1)
+    .background_gradient(
+        subset=["p-value of Corr."],
+        cmap="Reds",
+        vmin=0.1,
+        vmax=1,
+    )
+    .format(subset=["Z Score", corr_name, "p-value of Corr."], formatter="{:.2}")
 )
 
+print(data["Benchmark"].unique().tolist())
 
 st.dataframe(
     data=styled_data,
     hide_index=True,
     use_container_width=True,
-    height=300,
+    height=500,
 )
 
 st.markdown(
@@ -309,7 +486,9 @@ st.write(r"""
 
 benchmarks = data["Benchmark"].unique().tolist()
 plotted_scenario = st.selectbox(
-    "Choose Benchmark to plot", benchmarks, index=benchmarks.index("LMSys Arena")
+    "Choose Benchmark to plot",
+    benchmarks,
+    index=benchmarks.index("Arena Elo"),
 )