added leaderboard data; added Class coloring

.gitignore

@@ -0,0 +1,50 @@
+/**/*_bkup.*
+# Python
+__pycache__
+.ipynb_checkpoints/
+*.pyc
+*.egg-info
+dist
+.venv
+
+# Log
+*.log
+*.log.*
+logs/controller/*
+logs/conversation/*
+logs/gradio_web_server/*
+logs/gradio_web_server_multi/*
+!logs/**/.keep
+*.json
+!playground/deepspeed_config_s2.json
+!playground/deepspeed_config_s3.json
+
+# Editor
+.idea
+*.swp
+
+# Other
+.DS_Store
+wandb
+output
+checkpoints_flant5_3b
+
+# Data
+*.pkl
+tests/state_of_the_union.txt
+
+# Build
+build
+
+# Gradio Temp
+tmp
+
+# API KEYS
+set_api_keys.sh
+
+# data
+
+
+!**/.keep
+
+archive/*

app.py

@@ -1,158 +1,282 @@
 import abc
 
 import gradio as gr
+from loguru import logger
+import pandas as pd
+from collections import defaultdict
+
+from judgerbench.preprocess.gen_table import (
+    format_timestamp,
+    generate_table, 
+    build_l1_df,
+    # build_l2_df,
+)
+from judgerbench.meta_data import (
+    LEADERBORAD_INTRODUCTION,
+    LEADERBOARD_MD,
+    LEADERBOARD_FILE_MAPPING,
+    MAIN_FIELDS,
+    DEFAULT_BENCH,
+    STYLE_CLASS_MAPPING,
+    CITATION_BUTTON_TEXT,
+    CITATION_BUTTON_LABEL,
+)
+
+
+def refresh_dataframe(required_fields):
+    df = generate_table(MAIN_FIELDS)
+
+    comp = gr.DataFrame(
+        value=df,
+        type='pandas',
+        interactive=False,
+        visible=True
+    )
+
+    return comp
 
-from gen_table import *
-from meta_data import *
 
 with gr.Blocks() as demo:
-    struct = load_results()
-    timestamp = struct['time']
-    EVAL_TIME = format_timestamp(timestamp)
-    results = struct['results']
-    N_MODEL = len(results)
-    N_DATA = len(results['LLaVA-v1.5-7B']) - 1
-    DATASETS = list(results['LLaVA-v1.5-7B'])
-    DATASETS.remove('META')
-    print(DATASETS)
-
-    gr.Markdown(LEADERBORAD_INTRODUCTION.format(N_MODEL, N_DATA, EVAL_TIME))
-    structs = [abc.abstractproperty() for _ in range(N_DATA)]
+    # struct = load_results()
+    # timestamp = struct['time']
+
+    # EVAL_TIME = format_timestamp(timestamp)
+    EVAL_TIME = '20241015'
+
+    # results = struct['results']
+    # N_MODEL = len(results)
+    # N_DATA = len(results['LLaVA-v1.5-7B']) - 1
+
+    N_MODEL = 10
+    N_DATA = 100
+
+    # DATASETS = list(results['LLaVA-v1.5-7B'])
+    # DATASETS.remove('META')
+    # print(DATASETS)
+
+    gr.Markdown(LEADERBORAD_INTRODUCTION.format(
+        # N_MODEL, 
+        # N_DATA, 
+        EVAL_TIME
+    ))
+    # structs = [abc.abstractproperty() for _ in range(N_DATA)]
 
     with gr.Tabs(elem_classes='tab-buttons') as tabs:
-        with gr.TabItem('🏅 OpenVLM Main Leaderboard', elem_id='main', id=0):
-            gr.Markdown(LEADERBOARD_MD['MAIN'])
-            _, check_box = BUILD_L1_DF(results, MAIN_FIELDS)
-            table = generate_table(results, DEFAULT_BENCH)
-            table['Rank'] = list(range(1, len(table) + 1))
-
-            type_map = check_box['type_map']
-            type_map['Rank'] = 'number'
-
-            checkbox_group = gr.CheckboxGroup(
-                choices=check_box['all'],
-                value=check_box['required'],
-                label='Evaluation Dimension',
-                interactive=True,
-            )
-
-            headers = ['Rank'] + check_box['essential'] + checkbox_group.value
-            with gr.Row():
-                model_size = gr.CheckboxGroup(
-                    choices=MODEL_SIZE,
-                    value=MODEL_SIZE,
-                    label='Model Size',
-                    interactive=True
-                )
-                model_type = gr.CheckboxGroup(
-                    choices=MODEL_TYPE,
-                    value=MODEL_TYPE,
-                    label='Model Type',
-                    interactive=True
+        for cur_id, (filename, filepath) in enumerate(LEADERBOARD_FILE_MAPPING.items()):
+
+            tab_name = filename
+            if filename == "overall":
+                tab_name = '🏅 JudgerBench Main Leaderboard'
+
+            with gr.Tab(tab_name, elem_id=f'tab_{cur_id}', id=cur_id):
+
+                # gr.Markdown(LEADERBOARD_MD['MAIN'])
+                # _, check_box = build_l1_df(MAIN_FIELDS)
+                table = generate_table(filename=filename)
+
+                # type_map = check_box['type_map']
+                type_map = defaultdict(lambda: 'number')
+                type_map['Model'] = 'str'
+                type_map['Class'] = 'str'
+                type_map['Rank'] = 'number'
+
+                # required_fields = gr.State(
+                #     check_box['essential'] 
+                #     # + ["Average"]
+                # )
+
+                # checkbox_group = gr.CheckboxGroup(
+                #     choices=[item for item in check_box['all'] if item not in required_fields.value],
+                #     value=[item for item in check_box['default'] if item not in required_fields.value],
+                #     label='Evaluation Metrics',
+                #     interactive=True,
+                # )
+
+                # headers = (
+                #     ['Rank'] +
+                #     required_fields.value +
+                #     [item for item in check_box['all'] if item not in required_fields.value]
+                #     # checkbox_group.value
+                # )
+
+                table['Rank'] = list(range(1, len(table) + 1))
+
+                # Rearrange columns
+                if "Class" in table.columns:
+                    starting_columns = ["Rank", "Models", "Class"]
+                else:
+                    starting_columns = ["Rank", "Models"]
+
+                table = table[starting_columns + [ col for col in table.columns if col not in starting_columns ]]
+
+                headers = (
+                    # ['Rank'] +
+                    list(table.columns)
                 )
-            data_component = gr.components.DataFrame(
-                value=table[headers],
-                type='pandas',
-                datatype=[type_map[x] for x in headers],
-                interactive=False,
-                visible=True)
-
-            def filter_df(fields, model_size, model_type):
-                filter_list = ['Avg Score', 'Avg Rank', 'OpenSource', 'Verified']
-                headers = ['Rank'] + check_box['essential'] + fields
-
-                new_fields = [field for field in fields if field not in filter_list]
-                df = generate_table(results, new_fields)
-
-                df['flag'] = [model_size_flag(x, model_size) for x in df['Param (B)']]
-                df = df[df['flag']]
-                df.pop('flag')
-                if len(df):
-                    df['flag'] = [model_type_flag(df.iloc[i], model_type) for i in range(len(df))]
-                    df = df[df['flag']]
-                    df.pop('flag')
-                df['Rank'] = list(range(1, len(df) + 1))
-
-                comp = gr.components.DataFrame(
-                    value=df[headers],
+
+                if "Class" in table.columns:
+                    def cell_styler(v):
+                        df = v.copy()
+
+                        class_var = df[['Class']].copy()
+
+                        df.loc[:, :] = ''
+                        df[['Class']] = class_var.map(lambda x: f"background-color: {STYLE_CLASS_MAPPING[x]}")
+                        logger.info(df['Class'])
+
+                        return df
+
+                    table_styler = (
+                        table.style.apply(cell_styler, axis=None)
+                        .format(precision=3)
+                    )
+                else:
+                    table_styler = table.style.format(prevision=3)
+
+                # with gr.Row():
+                #     model_size = gr.CheckboxGroup(
+                #         choices=MODEL_SIZE,
+                #         value=MODEL_SIZE,
+                #         label='Model Size',
+                #         interactive=True
+                #     )
+                #     model_type = gr.CheckboxGroup(
+                #         choices=MODEL_TYPE,
+                #         value=MODEL_TYPE,
+                #         label='Model Type',
+                #         interactive=True
+                #     )
+                data_component = gr.DataFrame(
+                    value=table_styler,
                     type='pandas',
                     datatype=[type_map[x] for x in headers],
                     interactive=False,
-                    visible=True)
-                return comp
-
-            for cbox in [checkbox_group, model_size, model_type]:
-                cbox.change(fn=filter_df, inputs=[checkbox_group, model_size, model_type], outputs=data_component)
-
-        with gr.TabItem('🔍 About', elem_id='about', id=1):
-            gr.Markdown(urlopen(VLMEVALKIT_README).read().decode())
-
-        for i, dataset in enumerate(DATASETS):
-            with gr.TabItem(f'📊 {dataset} Leaderboard', elem_id=dataset, id=i + 2):
-                if dataset in LEADERBOARD_MD:
-                    gr.Markdown(LEADERBOARD_MD[dataset])
-
-                s = structs[i]
-                s.table, s.check_box = BUILD_L2_DF(results, dataset)
-                s.type_map = s.check_box['type_map']
-                s.type_map['Rank'] = 'number'
-
-                s.checkbox_group = gr.CheckboxGroup(
-                    choices=s.check_box['all'],
-                    value=s.check_box['required'],
-                    label=f'{dataset} CheckBoxes',
-                    interactive=True,
+                    visible=True
                 )
-                s.headers = ['Rank'] + s.check_box['essential'] + s.checkbox_group.value
-                s.table['Rank'] = list(range(1, len(s.table) + 1))
-
-                with gr.Row():
-                    s.model_size = gr.CheckboxGroup(
-                        choices=MODEL_SIZE,
-                        value=MODEL_SIZE,
-                        label='Model Size',
-                        interactive=True
-                    )
-                    s.model_type = gr.CheckboxGroup(
-                        choices=MODEL_TYPE,
-                        value=MODEL_TYPE,
-                        label='Model Type',
-                        interactive=True
-                    )
-                s.data_component = gr.components.DataFrame(
-                    value=s.table[s.headers],
-                    type='pandas',
-                    datatype=[s.type_map[x] for x in s.headers],
-                    interactive=False,
-                    visible=True)
-                s.dataset = gr.Textbox(value=dataset, label=dataset, visible=False)
-
-                def filter_df_l2(dataset_name, fields, model_size, model_type):
-                    s = structs[DATASETS.index(dataset_name)]
-                    headers = ['Rank'] + s.check_box['essential'] + fields
-                    df = cp.deepcopy(s.table)
-                    df['flag'] = [model_size_flag(x, model_size) for x in df['Param (B)']]
-                    df = df[df['flag']]
-                    df.pop('flag')
-                    if len(df):
-                        df['flag'] = [model_type_flag(df.iloc[i], model_type) for i in range(len(df))]
-                        df = df[df['flag']]
-                        df.pop('flag')
+
+                def filter_df(
+                        required_fields,
+                        fields,
+                        # model_size, 
+                        # model_type
+                    ):
+                    # filter_list = ['Avg Score', 'Avg Rank', 'OpenSource', 'Verified']
+                    headers = ['Rank'] + required_fields + fields
+
+                    # new_fields = [field for field in fields if field not in filter_list]
+                    df = generate_table(fields)
+                    logger.info(f"{df.columns=}")
+
+                    # df['flag'] = [model_size_flag(x, model_size) for x in df['Param (B)']]
+                    # df = df[df['flag']]
+                    # df.pop('flag')
+
+                    # if len(df):
+                    #     df['flag'] = [model_type_flag(df.iloc[i], model_type) for i in range(len(df))]
+                    #     df = df[df['flag']]
+                    #     df.pop('flag')
+
                     df['Rank'] = list(range(1, len(df) + 1))
 
-                    comp = gr.components.DataFrame(
+                    comp = gr.DataFrame(
                         value=df[headers],
                         type='pandas',
-                        datatype=[s.type_map[x] for x in headers],
+                        datatype=[type_map[x] for x in headers],
                         interactive=False,
-                        visible=True)
+                        visible=True
+                    )
+                    
                     return comp
 
-                for cbox in [s.checkbox_group, s.model_size, s.model_type]:
-                    cbox.change(
-                        fn=filter_df_l2,
-                        inputs=[s.dataset, s.checkbox_group, s.model_size, s.model_type],
-                        outputs=s.data_component)
+                # for cbox in [
+                #         # checkbox_group, 
+                #         # model_size, 
+                #         # model_type
+                #     ]:
+                #     cbox.change(
+                #         fn=refresh_dataframe,
+                #         inputs=[required_fields],
+                #         outputs=data_component
+                #     ).then(
+                #         fn=filter_df,
+                #         inputs=[
+                #             required_fields,
+                #             checkbox_group, 
+                #             # model_size, 
+                #             # model_type
+                #         ], 
+                #         outputs=data_component
+                #     )
+
+            # with gr.Tab('🔍 About', elem_id='about', id=1):
+            #     gr.Markdown(urlopen(VLMEVALKIT_README).read().decode())
+
+            # for i, dataset in enumerate(DATASETS):
+            #     with gr.Tab(f'📊 {dataset} Leaderboard', elem_id=dataset, id=i + 2):
+            #         if dataset in LEADERBOARD_MD:
+            #             gr.Markdown(LEADERBOARD_MD[dataset])
+
+            #         s = structs[i]
+            #         s.table, s.check_box = build_l2_df(results, dataset)
+            #         s.type_map = s.check_box['type_map']
+            #         s.type_map['Rank'] = 'number'
+
+            #         s.checkbox_group = gr.CheckboxGroup(
+            #             choices=s.check_box['all'],
+            #             value=s.check_box['required'],
+            #             label=f'{dataset} CheckBoxes',
+            #             interactive=True,
+            #         )
+            #         s.headers = ['Rank'] + s.check_box['essential'] + s.checkbox_group.value
+            #         s.table['Rank'] = list(range(1, len(s.table) + 1))
+
+            #         with gr.Row():
+            #             s.model_size = gr.CheckboxGroup(
+            #                 choices=MODEL_SIZE,
+            #                 value=MODEL_SIZE,
+            #                 label='Model Size',
+            #                 interactive=True
+            #             )
+            #             s.model_type = gr.CheckboxGroup(
+            #                 choices=MODEL_TYPE,
+            #                 value=MODEL_TYPE,
+            #                 label='Model Type',
+            #                 interactive=True
+            #             )
+            #         s.data_component = gr.components.DataFrame(
+            #             value=s.table[s.headers],
+            #             type='pandas',
+            #             datatype=[s.type_map[x] for x in s.headers],
+            #             interactive=False,
+            #             visible=True)
+            #         s.dataset = gr.Textbox(value=dataset, label=dataset, visible=False)
+
+            #         def filter_df_l2(dataset_name, fields, model_size, model_type):
+            #             s = structs[DATASETS.index(dataset_name)]
+            #             headers = ['Rank'] + s.check_box['essential'] + fields
+            #             df = cp.deepcopy(s.table)
+            #             df['flag'] = [model_size_flag(x, model_size) for x in df['Param (B)']]
+            #             df = df[df['flag']]
+            #             df.pop('flag')
+            #             if len(df):
+            #                 df['flag'] = [model_type_flag(df.iloc[i], model_type) for i in range(len(df))]
+            #                 df = df[df['flag']]
+            #                 df.pop('flag')
+            #             df['Rank'] = list(range(1, len(df) + 1))
+
+            #             comp = gr.components.DataFrame(
+            #                 value=df[headers],
+            #                 type='pandas',
+            #                 datatype=[s.type_map[x] for x in headers],
+            #                 interactive=False,
+            #                 visible=True)
+            #             return comp
+
+            #         for cbox in [s.checkbox_group, s.model_size, s.model_type]:
+            #             cbox.change(
+            #                 fn=filter_df_l2,
+            #                 inputs=[s.dataset, s.checkbox_group, s.model_size, s.model_type],
+            #                 outputs=s.data_component)
 
     with gr.Row():
         with gr.Accordion('Citation', open=False):
@@ -161,5 +285,73 @@ with gr.Blocks() as demo:
                 label=CITATION_BUTTON_LABEL,
                 elem_id='citation-button')
 
+
 if __name__ == '__main__':
-    demo.launch(server_name='0.0.0.0')
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", type=str, default="0.0.0.0")
+    parser.add_argument("--port", type=int)
+    parser.add_argument(
+        "--share",
+        action="store_true",
+        help="Whether to generate a public, shareable link",
+    )
+    parser.add_argument(
+        "--concurrency-count",
+        type=int,
+        default=10,
+        help="The concurrency count of the gradio queue",
+    )
+    parser.add_argument(
+        "--max-threads",
+        type=int,
+        default=200,
+        help="The maximum number of threads available to process non-async functions.",
+    )
+    # parser.add_argument(
+    #     "--gradio-auth-path",
+    #     type=str,
+    #     help='Set the gradio authentication file path. The file should contain one or more user:password pairs in this format: "u1:p1,u2:p2,u3:p3"',
+    #     default=None,
+    # )
+    parser.add_argument(
+        "--gradio-root-path",
+        type=str,
+        help="Sets the gradio root path, eg /abc/def. Useful when running behind a reverse-proxy or at a custom URL path prefix",
+    )
+    parser.add_argument(
+        "--ga-id",
+        type=str,
+        help="the Google Analytics ID",
+        default=None,
+    )
+    parser.add_argument(
+        "--use-remote-storage",
+        action="store_true",
+        default=False,
+        help="Uploads image files to google cloud storage if set to true",
+    )
+    args = parser.parse_args()
+    logger.info(f"args: {args}")
+
+    # Set authorization credentials
+    # auth = None
+    # if args.gradio_auth_path is not None:
+    #     auth = parse_gradio_auth_creds(args.gradio_auth_path)
+
+    demo.queue(
+        default_concurrency_limit=args.concurrency_count,
+        status_update_rate=10,
+        api_open=False,
+    ).launch(
+        server_name=args.host,
+        server_port=args.port,
+        share=args.share,
+        max_threads=args.max_threads,
+        # auth=auth,
+        root_path=args.gradio_root_path,
+        # debug=True,
+        show_error=True,
+        allowed_paths=["../.."]
+    )

commands.sh

@@ -0,0 +1,12 @@
+srun \
+    --partition=llmeval \
+    --quotatype=reserved \
+    --job-name=judgerbench_server \
+    --ntasks=1 \
+    --ntasks-per-node=1 \
+    --cpus-per-task=2 \
+    --kill-on-bad-exit=1 \
+    --pty bash
+
+
+bash -i start_gradio_web_server.sh

data/detail_a_cn.csv

@@ -0,0 +1,11 @@
+Models,Teaser,AI,Roleplay,Chat,Math,Reasoning,Creation,Code,Science,Humanities,Average,Class
+CJ-1-14B,0.69,0.61,0.51,0.55,0.71,0.68,0.6,0.58,0.61,0.65,0.619,Judge
+GPT-4o-0806,0.77,0.56,0.51,0.53,0.67,0.66,0.63,0.58,0.62,0.58,0.611,API
+CJ-1-32B,0.69,0.58,0.53,0.52,0.71,0.53,0.6,0.61,0.61,0.69,0.607,Judge
+Skywork-llama3.1-8B,0.62,0.58,0.58,0.59,0.63,0.58,0.6,0.61,0.6,0.61,0.6,Judge
+Qwen2.5-72B-Chat,0.65,0.47,0.49,0.47,0.71,0.6,0.57,0.58,0.69,0.6,0.583,General
+CJ-1-7B,0.62,0.54,0.41,0.58,0.7,0.6,0.59,0.56,0.59,0.6,0.579,Judge
+Qwen2-72B-Chat,0.62,0.54,0.34,0.55,0.68,0.63,0.58,0.58,0.62,0.64,0.578,General
+Selftaught-llama3.1-70B,0.62,0.56,0.55,0.48,0.67,0.55,0.57,0.57,0.51,0.61,0.569,Judge
+Qwen2.5-7B-Chat,0.46,0.58,0.36,0.45,0.7,0.53,0.52,0.53,0.52,0.64,0.529,General
+CJ-1-1.5B,0.54,0.58,0.38,0.38,0.62,0.63,0.54,0.52,0.55,0.54,0.528,Judge

data/detail_a_en.csv

@@ -0,0 +1,11 @@
+Models,Teaser,AI,Roleplay,Chat,Math,Reasoning,Creation,Code,Science,Humanities,Average,Class
+GPT-4o-0806,0.82,0.53,0.62,0.61,0.83,0.67,0.67,0.73,0.64,0.55,0.667,API
+Skywork-llama3.1-8B,0.69,0.61,0.54,0.62,0.63,0.64,0.6,0.69,0.74,0.53,0.629,Judge
+Qwen2.5-72B-Chat,0.68,0.57,0.57,0.47,0.78,0.64,0.58,0.75,0.61,0.52,0.617,General
+CJ-1-32B,0.66,0.57,0.56,0.59,0.78,0.58,0.55,0.75,0.6,0.49,0.613,Judge
+CJ-1-14B,0.66,0.51,0.57,0.54,0.72,0.61,0.56,0.74,0.61,0.47,0.599,Judge
+Qwen2-72B-Chat,0.63,0.59,0.54,0.49,0.62,0.64,0.6,0.74,0.51,0.52,0.588,General
+CJ-1-7B,0.56,0.56,0.51,0.47,0.68,0.58,0.58,0.75,0.58,0.43,0.57,Judge
+Qwen2.5-7B-Chat,0.54,0.59,0.59,0.46,0.69,0.43,0.61,0.65,0.58,0.52,0.566,General
+CJ-1-1.5B,0.42,0.56,0.56,0.43,0.66,0.47,0.55,0.78,0.64,0.44,0.551,Judge
+Selftaught-llama3.1-70B,0.47,0.45,0.47,0.37,0.45,0.43,0.36,0.58,0.48,0.36,0.442,Judge

data/detail_b_acc.csv

@@ -0,0 +1,9 @@
+Models,AlignBench,Fofo,WildBench,ArenaHard,Average,Class
+CJ-1-32B,0.857,0.806,0.596,0.621,0.72,Judge
+CJ-1-14B,0.839,0.787,0.566,0.602,0.699,Judge
+CJ-1-7B,0.816,0.783,0.564,0.586,0.687,Judge
+Qwen2.5-72B-Chat,0.878,0.677,0.599,0.57,0.681,General
+CJ-1-1.5B,0.822,0.712,0.55,0.43,0.629,Judge
+Qwen2-72B-Chat,0.867,0.692,0.564,0.376,0.625,General
+Selftaught-llama3.1-70B,0.755,0.627,0.538,0.472,0.598,Judge
+Qwen2.5-7B-Chat,0.777,0.67,0.47,0.444,0.59,General

data/detail_b_corr.csv

@@ -0,0 +1,9 @@
+Models,AlignBench,Fofo,WildBench,ArenaHard,Average,Class
+CJ-1-32B,0.973,0.951,0.954,0.975,0.963,Judge
+CJ-1-14B,0.966,0.956,0.965,0.951,0.959,Judge
+CJ-1-7B,0.956,0.936,0.97,0.932,0.948,Judge
+Qwen2.5-72B-Chat,0.964,0.916,0.958,0.912,0.937,General
+Qwen2-72B-Chat,0.937,0.889,0.976,0.936,0.935,General
+CJ-1-1.5B,0.928,0.851,0.981,0.858,0.905,Judge
+Qwen2.5-7B-Chat,0.916,0.681,0.967,0.931,0.874,General
+Selftaught-llama3.1-70B,0.918,0.667,0.95,0.942,0.869,Judge

data/overall.csv

@@ -0,0 +1,11 @@
+Models,JDB-A EN,JDB-A CN,JDB-B Acc,JDB-B Corr,JudgerBench,Class
+GPT-4o-0806,0.664,0.608,1,1,0.818,API
+CJ-1-32B,0.614,0.612,0.72,0.963,0.727,Judge
+CJ-1-14B,0.599,0.615,0.699,0.959,0.718,Judge
+Qwen2.5-72B-Chat,0.615,0.59,0.681,0.937,0.706,General
+CJ-1-7B,0.57,0.583,0.687,0.948,0.697,Judge
+Qwen2-72B-Chat,0.588,0.584,0.625,0.935,0.683,General
+CJ-1-1.5B,0.553,0.527,0.629,0.905,0.654,Judge
+Qwen2.5-7B-Chat,0.567,0.535,0.59,0.874,0.641,General
+Selftaught-llama3.1-70B,0.443,0.57,0.598,0.869,0.62,Judge
+Skywork-llama3.1-8B,0.63,0.605,-,-,-,Judge

meta_data.py → judgerbench/meta_data.py

@@ -1,3 +1,25 @@
+from enum import IntEnum
+from pathlib import Path
+import os
+
+# REPO paths
+REPO_PATH = Path(os.path.dirname(os.path.dirname(__file__)))
+DATADIR = REPO_PATH / Path(os.getenv("DATADIR", "./data"))
+
+LEADERBOARD_FILE_MAPPING = dict(
+    overall="overall.csv",
+    detail_a_cn="detail_a_cn.csv",
+    detail_a_en="detail_a_en.csv",
+    detail_b_acc="detail_b_acc.csv",
+    detail_b_corr="detail_b_corr.csv",
+)
+
+STYLE_CLASS_MAPPING = {
+    "API": '#82e0aa',
+    "Judge": '#f8c471',
+    "General": '#85c1e9',
+}
+
 # CONSTANTS-URL
 URL = "http://opencompass.openxlab.space/assets/OpenVLM.json"
 VLMEVALKIT_README = 'https://raw.githubusercontent.com/open-compass/VLMEvalKit/main/README.md'
@@ -9,32 +31,59 @@ CITATION_BUTTON_TEXT = r"""@misc{2023opencompass,
     year={2023}
 }"""
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+
 # CONSTANTS-TEXT
-LEADERBORAD_INTRODUCTION = """# OpenVLM Leaderboard
-### Welcome to the OpenVLM Leaderboard! On this leaderboard we share the evaluation results of VLMs obtained by the OpenSource Framework:
-### [*VLMEvalKit*: A Toolkit for Evaluating Large Vision-Language Models](https://github.com/open-compass/VLMEvalKit) 🏆
-### Currently, OpenVLM Leaderboard covers {} different VLMs (including GPT-4v, Gemini, QwenVLPlus, LLaVA, etc.) and {} different multi-modal benchmarks. 
+LEADERBORAD_INTRODUCTION = """# JudgerBench Leaderboard
 
-This leaderboard was last updated: {}. 
+### Welcome to the JudgerBench Leaderboard!
 
-OpenVLM Leaderboard only includes open-source VLMs or API models that are publicly available. To add your own model to the leaderboard, please create a PR in [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) to support your VLM and then we will help with the evaluation and updating the leaderboard. For any questions or concerns, please feel free to contact us at [opencompass, duanhaodong]@pjlab.org.cn.
+This leaderboard was last updated: {}. 
 """
+
 # CONSTANTS-FIELDS
-META_FIELDS = ['Method', 'Param (B)', 'Language Model', 'Vision Model', 'OpenSource', 'Verified']
+# META_FIELDS = ['Method', 'Param (B)', 'Language Model', 'Vision Model', 'OpenSource', 'Verified']
+META_FIELDS = [
+    'Model',
+]
+# MAIN_FIELDS = [
+#     'MMBench_V11', 'MMStar', 'MME',
+#     'MMMU_VAL', 'MathVista', 'OCRBench', 'AI2D', 
+#     'HallusionBench', 'SEEDBench_IMG', 'MMVet', 
+#     'LLaVABench', 'CCBench', 'RealWorldQA', 'POPE', 'ScienceQA_TEST',
+#     'SEEDBench2_Plus', 'MMT-Bench_VAL', 'BLINK'
+# ]
 MAIN_FIELDS = [
-    'MMBench_V11', 'MMStar', 'MME',
-    'MMMU_VAL', 'MathVista', 'OCRBench', 'AI2D', 
-    'HallusionBench', 'SEEDBench_IMG', 'MMVet', 
-    'LLaVABench', 'CCBench', 'RealWorldQA', 'POPE', 'ScienceQA_TEST',
-    'SEEDBench2_Plus', 'MMT-Bench_VAL', 'BLINK'
+    'Average',
+    'Accuracy',
+    'Accuracy_CN',
+    'Accuracy_EN',
+    'Corr',
 ]
+# DEFAULT_BENCH = [
+#     'MMBench_V11', 'MMStar', 'MMMU_VAL', 'MathVista', 'OCRBench', 'AI2D', 
+#     'HallusionBench', 'MMVet'
+# ]
 DEFAULT_BENCH = [
-    'MMBench_V11', 'MMStar', 'MMMU_VAL', 'MathVista', 'OCRBench', 'AI2D', 
-    'HallusionBench', 'MMVet'
+    'Average',
+    'Accuracy',
+    'Accuracy_CN',
+    'Accuracy_EN',
+    'Corr',
 ]
-MMBENCH_FIELDS = ['MMBench_TEST_EN_V11', 'MMBench_TEST_CN_V11', 'MMBench_TEST_EN', 'MMBench_TEST_CN', 'CCBench']
-MODEL_SIZE = ['<4B', '4B-10B', '10B-20B', '20B-40B', '>40B', 'Unknown']
-MODEL_TYPE = ['API', 'OpenSource', 'Proprietary']
+
+FIELD_MAPPING = {
+    'model': 'Model',
+    'parameters': 'Param (B)',
+    'average': 'Average',
+    'accuracy': 'Accuracy',
+    'accuracy_cn': 'Accuracy_CN',
+    'accuracy_en': 'Accuracy_EN',
+    'corr': 'Corr',
+}
+
+# MMBENCH_FIELDS = ['MMBench_TEST_EN_V11', 'MMBench_TEST_CN_V11', 'MMBench_TEST_EN', 'MMBench_TEST_CN', 'CCBench']
+# MODEL_SIZE = ['<4B', '4B-10B', '10B-20B', '20B-40B', '>40B', 'Unknown']
+# MODEL_TYPE = ['API', 'OpenSource', 'Proprietary']
 
 # The README file for each benchmark
 LEADERBOARD_MD = {}

gen_table.py → judgerbench/preprocess/gen_table.py

@@ -6,9 +6,19 @@ from urllib.request import urlopen
 import gradio as gr
 import numpy as np
 import pandas as pd
+from pathlib import Path
+from typing import Union, List, Dict
+from loguru import logger
 
-from meta_data import DEFAULT_BENCH, META_FIELDS, URL
-
+from judgerbench.meta_data import (
+    DATADIR,
+    LEADERBOARD_FILE_MAPPING,
+    DEFAULT_BENCH,
+    FIELD_MAPPING,
+    STYLE_CLASS_MAPPING,
+    META_FIELDS, 
+    URL
+)
 
 def listinstr(lst, s):
     assert isinstance(lst, list)
@@ -18,7 +28,7 @@ def listinstr(lst, s):
     return False
 
 
-def load_results():
+def load_results_from_url():
     data = json.loads(urlopen(URL).read())
     return data
 
@@ -61,23 +71,35 @@ def model_type_flag(line, FIELDS):
     return False
 
 
-def BUILD_L1_DF(results, fields):
+def build_l1_df(fields):
     check_box = {}
-    check_box['essential'] = ['Method', 'Param (B)', 'Language Model', 'Vision Model']
+    check_box['essential'] = [
+        # 'Method', 
+        # 'Param (B)', 
+        'Model',
+    ]
     # revise there to set default dataset
-    check_box['required'] = ['Avg Score', 'Avg Rank'] + DEFAULT_BENCH
-    check_box['avg'] = ['Avg Score', 'Avg Rank']
-    check_box['all'] = check_box['avg'] + fields
+    check_box['default'] = DEFAULT_BENCH
+
+    check_box['avg'] = ['Average']
+    check_box['accuracy'] = ['Accuracy_CN', 'Accuracy_EN', 'Accuracy',]
+    check_box['all'] = fields
+
     type_map = defaultdict(lambda: 'number')
-    type_map['Method'] = 'html'
-    type_map['Language Model'] = type_map['Vision Model'] = type_map['OpenSource'] = type_map['Verified'] = 'str'
+    # type_map['Method'] = 'html'
+    type_map['Model'] = 'str'
+    # type_map['Language Model'] = 'str'
+    # type_map['Vision Model'] = 'str'
+    # type_map['OpenSource'] = 'str'
+    # type_map['Verified'] = 'str'
+
     check_box['type_map'] = type_map
 
-    df = generate_table(results, fields)
+    df = generate_table(fields)
     return df, check_box
 
 
-def BUILD_L2_DF(results, dataset):
+def build_l2_df(results, dataset):
     res = defaultdict(list)
     sub = [v for v in results.values() if dataset in v]
     assert len(sub)
@@ -137,7 +159,7 @@ def BUILD_L2_DF(results, dataset):
     return df, check_box
 
 
-def generate_table(results, fields):
+def generate_table1(results, fields):
 
     def get_mmbench_v11(item):
         assert 'MMBench_TEST_CN_V11' in item and 'MMBench_TEST_EN_V11' in item
@@ -194,3 +216,60 @@ def generate_table(results, fields):
         missing = missing.iloc[::-1]
     df = pd.concat([valid, missing])
     return df
+
+
+def generate_table(
+        fields: List[str] = None,
+        filename: str = None,
+        path: Union[str, Path] = DATADIR / "overall.csv",
+    ):
+
+    if filename in LEADERBOARD_FILE_MAPPING:
+        path = DATADIR / LEADERBOARD_FILE_MAPPING[filename]
+
+    if filename is None and path is None:
+        raise ValueError("filename and path cannot both be None.")
+
+    REQUIRED_FILEDS = META_FIELDS + [
+        # 'Average'
+    ]
+
+    df = pd.read_csv(path)
+    # df_reshaped = (
+    #     df
+    #     .drop(columns=["dataset", "mode", "version"])
+    #     .melt(
+    #         id_vars=["metric"], 
+    #         var_name="model", 
+    #         value_name="value"
+    #     )
+    #     .pivot(index=["model"], columns=["metric"], values='value')
+    # )
+    # df_reshaped.columns.name = None
+    # df_reshaped.reset_index(inplace=True)
+    # df_reshaped.rename(columns=FIELD_MAPPING, inplace=True)
+
+    # if fields is not None:
+    #     for field in fields:
+    #         if field not in df_reshaped.columns:
+    #             raise ValueError(f"{field} is not a valid field in leaderboard table.")
+        
+    # new_fields = [field for field in FIELD_MAPPING.values() if field in REQUIRED_FILEDS + fields]
+    # logger.info(f"{new_fields=}")
+
+    # df_reshaped = df_reshaped.loc[:,new_fields].copy()
+
+    # valid, missing = df_reshaped[~pd.isna(df_reshaped['Average'])], df_reshaped[pd.isna(df_reshaped['Average'])]
+    # valid = valid.sort_values('Average', ascending=False)
+
+    # if len(fields):
+    #     missing = missing.sort_values(
+    #         'Accuracy' if 'Accuracy' in fields else fields[0],
+    #         ascending=False,
+    #     )
+
+    # df_sorted = pd.concat([valid, missing])
+
+    df_sorted = df
+
+    return df_sorted

judgerbench/preprocess/generate_table.py

@@ -0,0 +1,27 @@
+#%%
+import pandas as pd
+
+from judgerbench.meta_data import (
+    DATADIR
+)
+
+#%%
+df = pd.read_csv(DATADIR / "summary_20241007_221023.csv")
+df
+
+# %%
+df_reshaped = (
+    df
+    .drop(columns=["dataset", "mode", "version"])
+    .melt(
+        id_vars=["metric"], 
+        var_name="model", 
+        value_name="value"
+    )
+    .pivot(index=["model"], columns=["metric"], values='value')
+)
+df_reshaped.columns.name = None
+df_reshaped.reset_index(inplace=True)
+
+df_reshaped
+# %%

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools >= 64"]
+build-backend = "setuptools.build_meta"

requirements.txt

@@ -1,3 +1,3 @@
-gradio==4.15.0
+gradio==5.1
 numpy>=1.23.4
 pandas>=1.5.3

setup.py

@@ -0,0 +1,53 @@
+"""Python setup.py for project_name package"""
+import io
+import os
+import re
+from setuptools import find_packages, setup
+    
+
+def read(*paths, **kwargs):
+    """Read the contents of a text file safely.
+    >>> read("project_name", "VERSION")
+    '0.1.0'
+    >>> read("README.md")
+    ...
+    """
+
+    content = ""
+    with io.open(
+        os.path.join(os.path.dirname(__file__), *paths),
+        encoding=kwargs.get("encoding", "utf8"),
+    ) as open_file:
+        content = open_file.read().strip()
+    return content
+
+
+def read_requirements(path):
+    return [
+        line.strip()
+        for line in read(path).split("\n")
+        if not line.startswith(('"', "#", "-", "git+"))
+    ]
+
+
+setup(
+    name="judgerbench",
+    version='v0.0.1',
+    description="Judger Bench",
+    url="https://huggingface.co/spaces/acylam/judgerbench_leaderboard",
+    long_description=read("README.md"),
+    # long_description_content_type="text/markdown",
+    author="linjunyao",
+    maintainer="linjunyao",
+    package_dir={"": "judgerbench"},
+    packages=find_packages(
+        where="judgerbench", 
+        include=["judgerbench", "judgerbench/**/*"], 
+        exclude=["tests", ".github"]
+    ),
+    install_requires=read_requirements("requirements.txt"),
+    # entry_points={
+    #     "console_scripts": ["project_name = project_name.__main__:main"]
+    # },
+    # extras_require={"test": read_requirements("requirements-test.txt")},
+)

start_gradio_web_server.sh

@@ -0,0 +1,23 @@
+conda activate judgerbench
+
+# GRADIO_HOSTNAME=0.0.0.0
+GRADIO_HOSTNAME=$(hostname)
+GRADIO_PORT=7861
+
+# Set tmp and logs folders for gradio
+export TMPDIR="tmp"
+export LOGDIR="logs"
+export GRADIO_SERVER_NAME="0.0.0.0"
+export no_proxy="$CONTROLLER_HOST,10.140.1.173,0.0.0.0,$(hostname)"
+
+# OpenAI proxy url
+# export OPENAI_PROXY_URL='http://10.1.20.57:23128'
+export OPENAI_PROXY_URL='http://closeai-proxy.pjlab.org.cn:23128'
+
+# Source api keys
+# source set_api_keys.sh
+
+python3 -m app \
+    --host $GRADIO_HOSTNAME \
+    --port $GRADIO_PORT \
+    --concurrency-count 50