Productin Commit

.gitattributes

@@ -25,7 +25,6 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
@@ -33,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,23 @@
+auto_evals/
+venv/
+hfvenv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+gpt_4_evals/
+human_evals/
+eval-queue/
+eval-results/
+eval-results-group/
+auto_evals/
+
+src/assets/model_counts.html
+
+test
+env
+a.py
+testing.py
+frontend

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -1,11 +1,36 @@
 ---
-title: Leaderboard Frontend
-emoji: 👀
-colorFrom: purple
-colorTo: purple
-sdk: static
-pinned: false
-license: cc-by-nc-sa-4.0
+title: Azerbaijani LLM Leaderboard
+emoji: 🥇
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+sdk_version: 4.36.1
+app_file: app.py
+pinned: true
+license: apache-2.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+Most of the variables to change for a default leaderboard are in env (replace the path for your leaderboard) and src/display/about.
+
+Results files should have the following format:
+```
+{
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
+}
+```
+
+Request files are created automatically by this tool.

app.py

@@ -0,0 +1,403 @@
+import gradio as gr
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
+import os
+os.environ['CURL_CA_BUNDLE'] = ''
+
+from src.display.about import (
+    EVALUATION_QUEUE_TEXT,
+    INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT,
+    LLM_DATASET_TEXT,
+    TITLE,
+)
+from src.display.css_html_js import custom_css
+from src.display.utils import (
+    BENCHMARK_COLS,
+    COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    TYPES,
+    AutoEvalColumn,
+    fields,
+    BENCHMARK_COLS_GROUP,
+    COLS_GROUP,
+    EVAL_COLS_GROUP,
+    EVAL_TYPES_GROUP,
+    TYPES_GROUP,
+    AutoEvalColumnGroup,
+)
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, TOKEN, QUEUE_REPO, REPO_ID, RESULTS_REPO, EVAL_RESULTS_GROUP_PATH, RESULTS_GROUP_REPO
+from src.populate import get_evaluation_queue_df, get_leaderboard_df, get_evaluation_queue_df_group, get_leaderboard_group_df
+from src.submission.submit import add_new_eval
+
+
+def restart_space():
+    API.restart_space(repo_id=REPO_ID, token=TOKEN)
+
+try:
+    print(EVAL_REQUESTS_PATH)
+    snapshot_download(
+        repo_id=QUEUE_REPO, 
+        local_dir=EVAL_REQUESTS_PATH, 
+        repo_type="dataset", 
+        tqdm_class=None, 
+        etag_timeout=30, 
+        force_download=True,
+        token=TOKEN
+    )
+except Exception:
+    restart_space()
+try:
+    print(EVAL_RESULTS_PATH)
+    snapshot_download(
+        repo_id=RESULTS_REPO, 
+        local_dir=EVAL_RESULTS_PATH, 
+        repo_type="dataset", 
+        tqdm_class=None, 
+        etag_timeout=30, 
+        force_download=True,
+        token=TOKEN
+    )
+    snapshot_download(
+        repo_id=RESULTS_GROUP_REPO, 
+        local_dir=EVAL_RESULTS_GROUP_PATH, 
+        repo_type="dataset", 
+        tqdm_class=None, 
+        etag_timeout=30, 
+        force_download=True,
+        token=TOKEN)
+except Exception:
+    restart_space()
+
+
+raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
+raw_data_grouped, original_df_grouped = get_leaderboard_group_df(EVAL_RESULTS_GROUP_PATH, COLS_GROUP, BENCHMARK_COLS_GROUP)
+
+leaderboard_grouped_df = original_df_grouped.copy()
+leaderboard_df = original_df.copy()
+
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+
+
+(
+    finished_eval_queue_g_df,
+    running_eval_queue_g_df,
+    pending_eval_queue_g_df,
+) = get_evaluation_queue_df_group(EVAL_REQUESTS_PATH, EVAL_COLS_GROUP)
+
+# Searching and filtering
+def update_table(
+    hidden_df: pd.DataFrame,
+    columns: list,
+    query: str,
+):
+    filtered_df = filter_queries(query, hidden_df)
+    df = select_columns(filtered_df, columns)
+    return df
+
+
+def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
+    return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
+
+
+def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
+    always_here_cols = [
+        AutoEvalColumn.model_submission_date.name,
+        AutoEvalColumn.model.name,
+    ]
+    # We use COLS to maintain sorting
+    filtered_df = df[
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]
+    ]
+    return filtered_df
+
+
+def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
+    final_df = []
+    if query != "":
+        queries = [q.strip() for q in query.split(";")]
+        for _q in queries:
+            if _q != "":
+                temp_filtered_df = search_table(filtered_df, _q)
+                if len(temp_filtered_df) > 0:
+                    final_df.append(temp_filtered_df)
+        if len(final_df) > 0:
+            filtered_df = pd.concat(final_df)
+            filtered_df = filtered_df.drop_duplicates(
+                subset=[AutoEvalColumn.model.name, AutoEvalColumn.model_submission_date.name]
+            )
+
+    return filtered_df
+
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    with gr.Row():
+        with gr.Column(scale=9):
+            gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+        with gr.Column(scale=1, min_width=1):
+            gr.Image('src/display/kapital.jpg', scale=1, 
+                    show_label=False,
+                    interactive=False,
+                    show_share_button=False,
+                    show_download_button=False)
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+            with gr.Row():
+                with gr.Row():
+                    search_bar = gr.Textbox(
+                        placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                        show_label=False,
+                        elem_id="search-bar",
+                        )
+                with gr.Row():
+                    shown_columns = gr.CheckboxGroup(
+                        choices=[
+                            c.name
+                            for c in fields(AutoEvalColumnGroup)
+                            if not c.hidden and not c.never_hidden and not c.dummy
+                        ],
+                        value=[
+                            c.name
+                            for c in fields(AutoEvalColumnGroup)
+                            if c.displayed_by_default and not c.hidden and not c.never_hidden
+                        ],
+                        label="Select columns to show",
+                        elem_id="column-select",
+                        interactive=True,
+                    )
+
+            leaderboard_table = gr.components.Dataframe(
+                value=leaderboard_grouped_df[
+                    [c.name for c in fields(AutoEvalColumnGroup) if c.never_hidden]
+                    + shown_columns.value
+                    + [AutoEvalColumnGroup.dummy.name]
+                ],
+                headers=[c.name for c in fields(AutoEvalColumnGroup) if c.never_hidden] + shown_columns.value + [AutoEvalColumnGroup.dummy.name],
+                datatype=TYPES_GROUP,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+                column_widths=["15%", "30%"]
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=original_df_grouped[COLS_GROUP],
+                headers=COLS_GROUP,
+                datatype=TYPES_GROUP,
+                visible=False,
+            )
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                ],
+                leaderboard_table,
+            )
+            for selector in [shown_columns]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+            
+        with gr.TabItem("🏅 LLM Benchmark FineGrained", elem_id="llm-benchmark-tab-table-1", id=1):
+            with gr.Row():
+                with gr.Row():
+                    search_bar = gr.Textbox(
+                        placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                        show_label=False,
+                        elem_id="search-bar",
+                        )
+                with gr.Row():
+                    shown_columns = gr.CheckboxGroup(
+                        choices=[
+                            c.name
+                            for c in fields(AutoEvalColumn)
+                            if not c.hidden and not c.never_hidden and not c.dummy
+                        ],
+                        value=[
+                            c.name
+                            for c in fields(AutoEvalColumn)
+                            if c.displayed_by_default and not c.hidden and not c.never_hidden
+                        ],
+                        label="Select columns to show",
+                        elem_id="column-select",
+                        interactive=True,
+                    )
+
+            leaderboard_table = gr.components.Dataframe(
+                    value=leaderboard_df[
+                        [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+                        + shown_columns.value
+                        + [AutoEvalColumn.dummy.name]
+                    ],
+                    headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value + [AutoEvalColumn.dummy.name],
+                    datatype=TYPES,
+                    elem_id="leaderboard-table",
+                    interactive=False,
+                    visible=True,
+                    column_widths=["15%", "30%"]
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=original_df[COLS],
+                headers=COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                ],
+                leaderboard_table,
+            )
+            for selector in [shown_columns]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=2):
+            with gr.Column():
+                with gr.Row():
+                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+                with gr.Column():
+                    with gr.Accordion(
+                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            finished_eval_table = gr.components.Dataframe(
+                                value=finished_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
+                            )
+                    with gr.Accordion(
+                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            running_eval_table = gr.components.Dataframe(
+                                value=running_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
+                            )
+
+                    with gr.Accordion(
+                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            pending_eval_table = gr.components.Dataframe(
+                                value=pending_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
+                            )
+
+                
+                
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        model_name_textbox = gr.Textbox(label="Model name")
+                        
+                with gr.Column():
+                    with gr.Row():
+                        weight_type = gr.Dropdown(
+                            choices=['safetensors', 'gguf'],
+                            label="Weights type",
+                            multiselect=False,
+                            value='safgit petensors',
+                            interactive=True,
+                        )
+                        
+                with gr.Column():
+                    with gr.Row():
+                        gguf_filename_textbox = gr.Textbox(label="GGUF filename")
+
+            submit_button = gr.Button("Submit Eval")
+            submission_result = gr.Markdown()
+            submit_button.click(
+                add_new_eval,
+                [
+                    model_name_textbox,
+                    weight_type,
+                    gguf_filename_textbox
+                ],
+                submission_result,
+            )
+                
+        with gr.TabItem("📝 Evaluation Datasets", elem_id="llm-benchmark-tab-table", id=4):
+            gr.Markdown(LLM_DATASET_TEXT, elem_classes="markdown-text")
+    gr.HTML("""<h1 align="center" id="space-title"> Contributor Companies and Teams </h1>""")
+    with gr.Row():
+        with gr.Column(scale=35):
+            pass
+        with gr.Column(scale=10,  min_width=1, elem_classes='center-column'):
+            gr.Image('src/display/localdocs.jpeg', 
+                    scale = 1, 
+                    height=160,
+                    show_label=False,
+                    interactive=False,
+                    show_share_button=False,
+                    show_download_button=False)
+            gr.HTML("""<h1 align="center" id="company tile"> LocalDocs </h1>""")
+        with gr.Column(scale=10,  min_width=1, elem_classes='center-column'):
+            gr.Image('src/display/prodata.png',  
+                    scale =  1, 
+                    height=160,
+                    show_label=False,
+                    interactive=False,
+                    show_share_button=False,
+                    show_download_button=False)
+            gr.HTML("""<h1 align="center" id="company tile"> PRODATA </h1>""")
+        with gr.Column(scale=10, min_width=1, elem_classes='center-column'):
+            gr.Image('src/display/bhosai.jpeg',
+                    scale = 1, 
+                    height=160,
+                    show_label=False,
+                    interactive=False,
+                    show_share_button=False,
+                    show_download_button=False)
+            gr.HTML("""<h1 align="center" id="company tile"> BHOSAI </h1>""")
+        with gr.Column(scale=35):
+            pass
+    
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1000)
+scheduler.start()
+demo.queue(default_concurrency_limit=40).launch()

index.html

@@ -1,19 +0,0 @@
-<!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
-</html>

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,88 @@
+aiofiles==23.2.1
+aiohappyeyeballs==2.4.2
+aiohttp==3.10.8
+aiosignal==1.3.1
+altair==5.4.1
+annotated-types==0.7.0
+anyio==4.6.0
+APScheduler==3.10.1
+async-timeout==4.0.3
+attrs==24.2.0
+black==23.11.0
+certifi==2024.8.30
+charset-normalizer==3.3.2
+click==8.1.3
+contourpy==1.3.0
+cycler==0.12.1
+datasets==2.14.5
+dill==0.3.7
+exceptiongroup==1.2.2
+fastapi==0.115.3
+ffmpy==0.4.0
+filelock==3.16.1
+fonttools==4.54.1
+frozenlist==1.4.1
+fsspec==2023.6.0
+gradio==5.3.0
+gradio_client==1.4.2
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.2
+huggingface-hub==0.25.1
+idna==3.10
+importlib_resources==6.4.5
+Jinja2==3.1.4
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.7
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.7.1
+mdurl==0.1.2
+multidict==6.1.0
+multiprocess==0.70.15
+mypy-extensions==1.0.0
+narwhals==1.8.4
+numpy==1.26.4
+orjson==3.10.7
+packaging==24.1
+pandas==2.0.0
+pathspec==0.12.1
+pillow==10.4.0
+platformdirs==4.3.6
+pyarrow==17.0.0
+pydantic==2.9.2
+pydantic_core==2.23.4
+pydub==0.25.1
+Pygments==2.18.0
+pyparsing==3.1.4
+python-dateutil==2.8.2
+python-multipart==0.0.12
+pytz==2024.2
+PyYAML==6.0.2
+referencing==0.35.1
+regex==2024.9.11
+requests==2.32.3
+rich==13.8.1
+rpds-py==0.20.0
+ruff==0.6.8
+safetensors==0.4.5
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+starlette==0.41.0
+tokenizers==0.15.2
+tomli==2.0.1
+tomlkit==0.12.0
+tqdm==4.65.0
+transformers==4.35.2
+typer==0.12.5
+typing_extensions==4.12.2
+tzdata==2024.2
+tzlocal==5.2
+urllib3==2.2.3
+uvicorn==0.31.0
+websockets==11.0.3
+xxhash==3.5.0
+yarl==1.13.1

src/datasets.json

@@ -0,0 +1,130 @@
+[
+    {
+        "task_type": "mmlu",
+        "dstype": "mc",
+        "group": "Banking",
+        "subtext": "You are an AI that selects the most accurate answer in Azerbaijani based on a given question. You will be provided with a question in Azerbaijani and multiple options in Azerbaijani. Choose the single letter (A, B, C, D) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/Banking_Exam_MCQ",
+        "name": "Banking_Exam_MCQ"
+    },
+    {
+        "task_type": "mmlu",
+        "dstype": "kmc_azerbaycan_dili",
+        "group": "MMLU",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani based on grammatical concepts and linguistics. Your task is to select the correct option from the given question and answer choices. Choose the single letter (A, B, C, D, E) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/Azerbaijani_Lang_MC",
+        "name": "Azerbaijani_Lang_MC"
+    },
+    {
+        "task_type": "mmlu",
+        "dstype": "kmc_edebiyyat",
+        "group": "MMLU",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani based on literary and historical facts. Your task is to select the correct option from the given question and answer choices. Choose the single letter (A, B, C, D, E) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/Literature_MC",
+        "name": "Azerbaijani_Lit_MC"
+    },
+    {
+        "task_type": "mmlu",
+        "dstype": "kmc_biologiya",
+        "group": "MMLU",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani based on biology. Your task is to select the correct option from the given question and answer choices. Choose the single letter (A, B, C, D, E) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/Biology_MC",
+        "name": "Biology_MC"
+    },
+    {
+        "task_type": "mmlu",
+        "dstype": "kmc_cografiya",
+        "group": "MMLU",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani based on geographical and environmental knowledge. Your task is to select the correct option from the given question and answer choices. Choose the single letter (A, B, C, D, E) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/Geography_MC",
+        "name": "Geography_MC"
+    },
+    {
+        "task_type": "mmlu",
+        "dstype": "kmc_mentiq",
+        "group": "MMLU",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani based on logical reasoning and problem-solving. Your task is to select the correct option from the given question and answer choices. Choose the single letter (A, B, C, D, E) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/Logic_MC",
+        "name": "Logic_MC"
+    },
+    {
+        "task_type": "mmlu",
+        "dstype": "kmc_tarix",
+        "group": "MMLU",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani based on historical and cultural facts. Your task is to select the correct option from the given question and answer choices. Choose the single letter (A, B, C, D, E) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/History_MC",
+        "name": "History_MC"
+    },
+    {
+        "task_type": "mmlu",
+        "dstype": "kmc_informatika",
+        "group": "MMLU",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani based on technology and computer science. Your task is to select the correct option from the given question and answer choices. Choose the single letter (A, B, C, D, E) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/Informatics_MC",
+        "name": "Informatics_MC"
+    },
+    {
+        "task_type": "mmlu",
+        "dstype": "kmc_fizika",
+        "group": "MMLU",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani based on physics concepts. Your task is to select the correct option from the given question and answer choices. Choose the single letter (A, B, C, D, E) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/Physics_MC",
+        "name": "Physics_MC"
+    },
+    {
+        "task_type": "mmlu",
+        "dstype": "kmc_kimya",
+        "group": "MMLU",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani based on chemistry and scientific concepts. Your task is to select the correct option from the given question and answer choices. Choose the single letter (A, B, C, D, E) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/Chemistry_MC",
+        "name": "Chemistry_MC"
+    },
+    {
+        "task_type": "mmlu",
+        "dstype": "kmc_azerbaycan_tarixi",
+        "group": "MMLU",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani based on historical facts. Your task is to select the correct option from the given question and answer choices. Choose the single letter (A, B, C, D, E) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/Azerbaijani_Hist_MC",
+        "name": "Azerbaijani_Hist_MC"
+    },
+    {
+        "task_type": "mmlu",
+        "dstype": "tc",
+        "group": "Banking",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani. Your task is to select the correct option from the given question and answer choices. You are given a statement along with multiple options that represent different topics. Choose the option that best categorizes the statement based on its topic. Choose the single letter (A, B, C, D, E, F, G, H, I, J) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/Banking_Call_Classification_MC",
+        "name": "Banking_Call_Classification_MC"
+    },
+    {
+        "task_type": "arc",
+        "dstype": "arc",
+        "group": "ARC",
+        "subtext": "You are an AI designed to answer questions in Azerbaijani based on reasoning and knowledge. Your task is to select the correct option from the given question and answer choices. You are given a question along with multiple options. Choose the correct option. Choose the single letter (A, B, C, D) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/ARC",
+        "name": "ARC"
+    },
+    {
+        "task_type": "gsm8k",
+        "dstype": "mmc",
+        "group": "GSM8K",
+        "subtext": "You are an AI designed to solve mathematical word problems in Azerbaijani. Your task is to analyze the given question and select the correct option from the provided choices. Choose the single letter (A, B, C, D, E) that best answers the question. Respond with only the letter of the chosen answer, without any additional text.",
+        "data": "LLM-Beetle/GSM8K",
+        "name": "GSM8K"
+    },
+    {
+        "task_type": "qa",
+        "dstype": "qa",
+        "group": "Banking",
+        "subtext": "",
+        "data": "LLM-Beetle/Banking_QA",
+        "name": "Banking_QA"
+    },
+    {
+        "task_type": "rag",
+        "dstype": "cqa",
+        "group": "CQA",
+        "subtext": "",
+        "data": "LLM-Beetle/Wiki_CQA",
+        "name": "Wiki_CQA"
+    }
+]

src/display/about.py

@@ -0,0 +1,103 @@
+from dataclasses import dataclass
+from enum import Enum
+import json
+
+
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+
+
+# Init: to update with your specific keys
+def create_task_list():
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    with open("src/datasets.json") as f:
+    
+        data = json.load(f)
+    
+    groups = []
+    names = []
+    for d in data:
+        groups.append(d['group'])
+        names.append(d['name'])
+    groups = list(set(groups))
+    tasks = []
+    grouped_tasks = []
+    for name in names:
+        tasks.append(Task(name, "metric_name", name))
+    for group in groups:
+        grouped_tasks.append(Task(group, "metric_name", group))
+    
+    return tasks, grouped_tasks
+    
+
+
+# Your leaderboard name
+TITLE = """<h1 align="center" id="space-title"> Azerbaijani LLM Leaderboard</h1>"""
+
+# What does your leaderboard evaluate?
+INTRODUCTION_TEXT = """
+## Azerbaijani Open LLM sponsored by Kapital Bank
+
+The Azerbaijani Open LLM Leaderboard is sponsored by Kapital Bank to support and develop Azerbaijani language NLP. This leaderboard offers a clear and fair ranking of open-source Azerbaijani LLMs, helping researchers, developers, and the AI community work together to improve the quality and use of Azerbaijani language models.
+Through this platform, we hope to bring useful AI technology to the Azerbaijani language and encourage models that are both locally relevant and internationally competitive.
+
+## Partners
+
+This leaderboard is supported by Kapital Bank, LocalDocs, PRODATA LLC, and the R&D Center of Baku Higher Oil School.
+
+"""
+
+LLM_BENCHMARKS_TEXT = f"""
+## Azerbaijani Open LLM sponsored by Kapital Bank
+
+The Azerbaijani Open LLM Leaderboard is sponsored by Kapital Bank to support and develop Azerbaijani language NLP. This leaderboard offers a clear and fair ranking of open-source Azerbaijani LLMs, helping researchers, developers, and the AI community work together to improve the quality and use of Azerbaijani language models.
+Through this platform, we hope to bring useful AI technology to the Azerbaijani language and encourage models that are both locally relevant and internationally competitive.PartnersThis leaderboard is supported by Kapital Bank, LocalDocs, PRODATA LLC, and the R&D Center of Baku Higher Oil School.
+
+"""
+
+LLM_DATASET_TEXT = f"""
+## Banking Call Classification MC: 
+
+192 entries; multiple-choice classification for bank-client requests.
+
+## Banking Exam MCQ: 
+
+200–300 multiple-choice questions based on university banking exam materials.
+
+## Banking QA: 97 entries: 
+ 
+Question-answer pairs on Azerbaijani banking topics.
+
+## Wiki CQA: 
+
+97 entries from Azerbaijani Wikipedia, with context, questions, and answers.
+
+## GSM8K: 
+
+44 grade-school math problems to test multi-step reasoning.
+
+## ARC:
+ 
+Elementary science questions in Azerbaijani, testing knowledge and reasoning.
+
+## Subject-Specific MCQs: 
+
+Questions across topics like informatics, history, physics, and more, each with 100 multiple-choice questions for specific subject knowledge.
+"""
+
+
+EVALUATION_QUEUE_TEXT = """
+## Some good practices before submitting a model
+
+### 1) Make sure your model exists on hub.
+### 2) Make sure your model is public.
+
+
+## In case of model failure
+If your model is displayed in the `FAILED` category, its execution stopped.
+Make sure you have followed the above steps first.
+Please contact us if you are facing any trouble!
+"""

src/display/css_html_js.py

@@ -0,0 +1,98 @@
+custom_css = """
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#models-to-add-text {
+    font-size: 18px !important;
+}
+
+#leaderboard-table {
+    margin-top: 15px
+}
+
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+}
+
+/* Hides the final AutoEvalColumn */
+#llm-benchmark-tab-table table td:last-child,
+#llm-benchmark-tab-table table th:last-child {
+    display: none;
+}
+
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+
+#scale-logo .download {
+    display: none;
+}
+#filter_type{
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span{
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap{
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner{  
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input{
+    width: 1px
+}
+#filter-columns-type{
+    border:0;
+    padding:0.5;
+}
+#filter-columns-size{
+    border:0;
+    padding:0.5;
+}
+#box-filter > .form{
+    border: 0
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/display/formatting.py

@@ -0,0 +1,36 @@
+import os
+from datetime import datetime, timezone
+
+from huggingface_hub import HfApi
+from huggingface_hub.hf_api import ModelInfo
+
+
+API = HfApi()
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -0,0 +1,94 @@
+from dataclasses import dataclass, make_dataclass
+
+from src.display.about import create_task_list
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+    dummy: bool = False
+
+Tasks, Groups = create_task_list()
+
+## Leaderboard columns
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model_submission_date", ColumnContent, ColumnContent("Submission Date", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+
+
+for task in Tasks:
+    auto_eval_column_dict.append([task.benchmark, ColumnContent, ColumnContent(task.col_name, "number", True)])
+# Dummy column for the search bar (hidden by the custom CSS)
+auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    submitted_time = ColumnContent("submitted_time", "str", True)
+    status = ColumnContent("status", "str", True)
+    
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [t.col_name for t in Tasks]
+
+
+
+
+
+#for grouping
+
+
+## Leaderboard columns
+auto_eval_group_dict = []
+# Init
+auto_eval_group_dict.append(["model_submission_date", ColumnContent, ColumnContent("Submission Date", "str", True, never_hidden=True)])
+auto_eval_group_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_group_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+
+
+for task in Groups:
+    auto_eval_group_dict.append([task.benchmark, ColumnContent, ColumnContent(task.col_name, "number", True)])
+# Dummy column for the search bar (hidden by the custom CSS)
+auto_eval_group_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumnGroup = make_dataclass("AutoEvalColumnGroup", auto_eval_group_dict, frozen=True)
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumnGroup:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    submitted_time = ColumnContent("submitted_time", "str", True)
+    status = ColumnContent("status", "str", True)
+    
+# Column selection
+COLS_GROUP = [c.name for c in fields(AutoEvalColumnGroup) if not c.hidden]
+TYPES_GROUP = [c.type for c in fields(AutoEvalColumnGroup) if not c.hidden]
+
+EVAL_COLS_GROUP = [c.name for c in fields(EvalQueueColumnGroup)]
+EVAL_TYPES_GROUP = [c.type for c in fields(EvalQueueColumnGroup)]
+
+BENCHMARK_COLS_GROUP = [t.col_name for t in Groups]

src/envs.py

@@ -0,0 +1,20 @@
+import os
+from huggingface_hub import HfApi
+
+# clone / pull the lmeh eval data
+TOKEN = os.environ.get("HF_TOKEN", None)
+
+OWNER = "LLM-Beetle"
+REPO_ID = f"{OWNER}/frontend"
+QUEUE_REPO = f"{OWNER}/requests"
+RESULTS_REPO = f"{OWNER}/results"
+RESULTS_GROUP_REPO = f"{OWNER}/grouped"
+
+CACHE_PATH=os.getenv("HF_HOME", ".")
+
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_RESULTS_GROUP_PATH = os.path.join(CACHE_PATH, "eval-results-group")
+
+API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -0,0 +1,225 @@
+import glob
+import json
+import math
+import os
+from dataclasses import dataclass
+
+import dateutil
+import numpy as np
+
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, Tasks, Groups
+
+@dataclass
+class EvalResult:
+    eval_name: str # org_model_date (uid)
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    results: dict
+    date: str = "" # submission date of request file
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+        # Get model and org
+        org_and_model = config.get("model_name", None)
+        org_and_model = org_and_model.split("/", 1)
+
+        org = org_and_model[0]
+        model = org_and_model[1]
+        date = config.get("submitted_time", None)
+        result_key = f"{org}_{model}_{date}"
+        full_model = "/".join(org_and_model)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Tasks:
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            date=date
+        )
+
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.model_submission_date.name: self.date,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.dummy.name: self.full_model,
+            AutoEvalColumn.average.name: average,
+        }
+
+        for task in Tasks:
+            data_dict[task.col_name] = self.results[task.benchmark]
+
+        return data_dict
+
+
+
+
+@dataclass
+class EvalResultGroup:
+    eval_name: str # org_model_date (uid)
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    results: dict
+    date: str = "" # submission date of request file
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+        # Get model and org
+        org_and_model = config.get("model_name", None)
+        org_and_model = org_and_model.split("/", 1)
+
+        org = org_and_model[0]
+        model = org_and_model[1]
+        date = config.get("submitted_time", None)
+        result_key = f"{org}_{model}_{date}"
+        full_model = "/".join(org_and_model)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Groups:
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            date=date
+        )
+
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        average = sum([v for v in self.results.values() if v is not None]) / len(Groups)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.model_submission_date.name: self.date,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.dummy.name: self.full_model,
+            AutoEvalColumn.average.name: average,
+        }
+
+        for task in Groups:
+            data_dict[task.col_name] = self.results[task.benchmark]
+
+        return data_dict
+
+
+
+
+
+
+
+
+def get_raw_eval_results(results_path: str) -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+
+    return results
+
+
+
+
+def get_group_eval_results(results_path: str) -> list[EvalResultGroup]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResultGroup.init_from_json_file(model_result_filepath)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        eval_results[eval_name] = eval_result
+
+    results = []
+    print(eval_results)
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            print("key error")
+            continue
+
+    return results
+

src/populate.py

@@ -0,0 +1,108 @@
+import json
+import os
+
+import pandas as pd
+
+from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, AutoEvalColumnGroup, EvalQueueColumnGroup
+from src.leaderboard.read_evals import get_raw_eval_results, get_group_eval_results
+
+
+def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    raw_data = get_raw_eval_results(results_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+
+    df = pd.DataFrame.from_records(all_data_json)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return raw_data, df
+
+
+def get_leaderboard_group_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    raw_data = get_group_eval_results(results_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+    df = pd.DataFrame.from_records(all_data_json)
+    df = df.sort_values(by=[AutoEvalColumnGroup.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return raw_data, df
+
+
+
+
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] == "PENDING"]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"] == "FINISHED"]
+    
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    
+    return df_finished[cols], df_running[cols], df_pending[cols]
+
+
+
+
+
+def get_evaluation_queue_df_group(save_path: str, cols: list) -> list[pd.DataFrame]:
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumnGroup.model.name] = make_clickable_model(data["model"])
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumnGroup.model.name] = make_clickable_model(data["model"])
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] == "PENDING"]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"] == "FINISHED"]
+    
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    
+    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/submit.py

@@ -0,0 +1,61 @@
+import json
+import os
+from datetime import datetime, timezone
+
+from src.display.formatting import styled_error, styled_message
+from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO
+
+
+def add_new_eval(model: str, weight_type: str, gguf_filename=None):
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model, revision='main')
+    except Exception:
+        return styled_error("Could not get your model information.")
+    
+    if weight_type=="safetensors":
+        if len(gguf_filename)!=0:
+            return styled_error("GGUF filename should be empty when using safetensors.")
+
+    # Seems good, creating the eval
+    print("Adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "weight_type": weight_type,
+        "gguf_filename": gguf_filename,
+        "status": "PENDING",
+        "submitted_time": current_time,
+    }
+
+    print("Creating eval file")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{current_time}.json"
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    print("Uploading eval file")
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # Remove the local file
+    os.remove(out_path)
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to five minutes for the model to show in the PENDING list."
+    )

style.css

@@ -1,28 +0,0 @@
-body {
-	padding: 2rem;
-	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
-}
-
-h1 {
-	font-size: 16px;
-	margin-top: 0;
-}
-
-p {
-	color: rgb(107, 114, 128);
-	font-size: 15px;
-	margin-bottom: 10px;
-	margin-top: 5px;
-}
-
-.card {
-	max-width: 620px;
-	margin: 0 auto;
-	padding: 16px;
-	border: 1px solid lightgray;
-	border-radius: 16px;
-}
-
-.card p:last-child {
-	margin-bottom: 0;
-}