revive leaderboard, fill with dummy data

app.py

@@ -1,9 +1,10 @@
 from functools import partial
 
 import gradio as gr
-# from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
+
 # from huggingface_hub import snapshot_download
 
 from src.about import (
@@ -17,27 +18,31 @@ from src.about import (
 from src.datamodel.data import F1Data
 
 from src.display.css_html_js import custom_css
-# from src.display.utils import (
-#     BENCHMARK_COLS,
-#     COLS,
-#     EVAL_COLS,
-#     EVAL_TYPES,
-#     AutoEvalColumn,
-#     ModelType,
-#     fields,
-#     WeightType,
-#     Precision
-# )
+
+from src.display.utils import (
+    # BENCHMARK_COLS,
+    COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    AutoEvalColumn,
+    ModelType,
+    fields,
+    WeightType,
+    Precision,
+)
 from src.envs import API, REPO_ID, TOKEN, CODE_PROBLEMS_REPO, SUBMISSIONS_REPO, RESULTS_REPO
 from src.logger import get_logger
-# from src.populate import get_evaluation_queue_df, get_leaderboard_df
+
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_solutions
 
 logger = get_logger(__name__)
 
+
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
+
 lbdb = F1Data(cp_ds_name=CODE_PROBLEMS_REPO, sub_ds_name=SUBMISSIONS_REPO, res_ds_name=RESULTS_REPO)
 
 logger.info("Initialized LBDB")
@@ -48,36 +53,35 @@ logger.info("Initialized LBDB")
 #     pending_eval_queue_df,
 # ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-# def init_leaderboard(dataframe):
-#     if dataframe is None or dataframe.empty:
-#         raise ValueError("Leaderboard DataFrame is empty or None.")
-#     return Leaderboard(
-#         value=dataframe,
-#         datatype=[c.type for c in fields(AutoEvalColumn)],
-#         select_columns=SelectColumns(
-#             default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-#             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-#             label="Select Columns to Display:",
-#         ),
-#         search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-#         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-#         filter_columns=[
-#             ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-#             ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-#             ColumnFilter(
-#                 AutoEvalColumn.params.name,
-#                 type="slider",
-#                 min=0.01,
-#                 max=150,
-#                 label="Select the number of parameters (B)",
-#             ),
-#             ColumnFilter(
-#                 AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-#             ),
-#         ],
-#         bool_checkboxgroup_label="Hide models",
-#         interactive=False,
-#     )
+
+def init_leaderboard(dataframe):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[AutoEvalColumn.system.name, AutoEvalColumn.system_type.name],
+        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.system_type.name, type="checkboxgroup", label="Model types"),
+            # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            # ColumnFilter(
+            #     AutoEvalColumn.params.name,
+            #     type="slider",
+            #     min=0.01,
+            #     max=150,
+            #     label="Select the number of parameters (B)",
+            # ),
+            # ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
 
 
 demo = gr.Blocks(css=custom_css)
@@ -86,8 +90,19 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        # with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-        #     leaderboard = init_leaderboard(LEADERBOARD_DF)
+        with gr.TabItem("🏅 FormulaOne Leaderboard", elem_id="formulaone-leaderboar-tab-table", id=0):
+            # TODO: activate
+            # leaderboard_df = get_leaderboard_df
+            # dummy df
+            leaderboard_df = pd.DataFrame(
+                {
+                    AutoEvalColumn.system.name: ["Model A", "Model B", "Model C"],  # AutoEvalColumn.model.name
+                    AutoEvalColumn.system_type.name: ["LLM", "LLM+Agent", "N/A"],  # AutoEvalColumn.model_type.name
+                    AutoEvalColumn.organization.name: ["Org A", "Org B", "Org C"],  # AutoEvalColumn.organization.name
+                    AutoEvalColumn.success_rate.name: [0.01, 0.0, 0.005],
+                }
+            )
+            leaderboard = init_leaderboard(leaderboard_df)
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=1):
             logger.info("Tab about")
@@ -149,7 +164,7 @@ with demo:
                     #     interactive=True,
                     # )
 
-                # with gr.Column():
+                    # with gr.Column():
                     submission_file = gr.File(label="JSONL solutions file", file_types=[".jsonl"])
                     # precision = gr.Dropdown(
                     #     choices=[i.value.name for i in Precision if i != Precision.Unknown],
@@ -170,8 +185,10 @@ with demo:
             logger.info("Submut button")
             submit_button = gr.Button("Submit")
             submission_result = gr.Markdown()
+
             def add_solution_cbk(submitter, submission_path):
                 return add_new_solutions(lbdb, submitter, submission_path)
+
             submit_button.click(
                 add_solution_cbk,
                 [

src/about.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from enum import Enum
 
+
 @dataclass
 class Task:
     benchmark: str
@@ -8,17 +9,17 @@ class Task:
     col_name: str
 
 
-# # Select your tasks here
-# # ---------------------------------------------------
-# class Tasks(Enum):
-#     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-#     task0 = Task("anli_r1", "acc", "ANLI")
-#     task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-NUM_FEWSHOT = 0 # Change with your few shot
+# Select your tasks here
 # ---------------------------------------------------
+class Tasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    task0 = Task("FormulaOne", "success_rate", "Success Rate (%)")
+    # task1 = Task("logiqa", "acc_norm", "LogiQA")
 
 
+NUM_FEWSHOT = 0  # Change with your few shot
+# ---------------------------------------------------
+
 
 # Your leaderboard name
 TITLE = """<h1 align="center" id="space-title">AAI FormulaOne Leaderboard</h1>"""

src/display/utils.py

@@ -1,10 +1,12 @@
-from dataclasses import dataclass, make_dataclass
+from dataclasses import dataclass, field, make_dataclass
+from typing import ClassVar
 from enum import Enum
 
 import pandas as pd
 
 from src.about import Tasks
 
+
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
@@ -20,28 +22,49 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
 
+
 ## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+# auto_eval_column_fields = []
+# # Init
+# auto_eval_column_fields.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+# auto_eval_column_fields.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+# # Scores
+# auto_eval_column_fields.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+# for task in Tasks:
+#     auto_eval_column_fields.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# # Model information
+# auto_eval_column_fields.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+# auto_eval_column_fields.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+# auto_eval_column_fields.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+# auto_eval_column_fields.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+# auto_eval_column_fields.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+# auto_eval_column_fields.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+# auto_eval_column_fields.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+# auto_eval_column_fields.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+# auto_eval_column_fields.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+#
+#
+#
+# def make_classvar_dataclass(name: str, spec: list):
+#     ns = {"__annotations__": {}}
+#     for field_name, field_type, default in spec:
+#         # Mark as ClassVar so dataclass doesn't treat it as an instance field
+#         ns["__annotations__"][field_name] = ClassVar[field_type]
+#         ns[field_name] = default
+#     # No instance fields; just class-level descriptors
+#     return make_dataclass(name, [], frozen=True, namespace=ns)
+#
+# # We use make dataclass to dynamically fill the scores from Tasks
+# AutoEvalColumn = make_classvar_dataclass("AutoEvalColumn", auto_eval_column_fields)
+
+@dataclass(frozen=True)
+class AutoEvalColumn:
+    system = ColumnContent("System Name", "markdown", True, never_hidden=True)
+    system_type = ColumnContent("System Type", "str", True)
+    organization = ColumnContent("Organization", "str", True, never_hidden=True)
+    success_rate = ColumnContent("Success Rate (%)", "number", True)
+
+
 
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
@@ -53,12 +76,13 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
+
 ## All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
     display_name: str = ""
-    symbol: str = "" # emoji
+    symbol: str = ""  # emoji
 
 
 class ModelType(Enum):
@@ -83,11 +107,13 @@ class ModelType(Enum):
             return ModelType.IFT
         return ModelType.Unknown
 
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
 
+
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
@@ -100,11 +126,11 @@ class Precision(Enum):
             return Precision.bfloat16
         return Precision.Unknown
 
+
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-
+# BENCHMARK_COLS = [t.value.col_name for t in Tasks]