Refactor leaderboard

- Update artifact handling and display logic
- Modify model submission process and evaluation requests
- Refactor data structures and file paths for Solidity-specific use case
- Remove unnecessary fields and simplify UI components
- Update environment variables and repository structure

.gitignore

@@ -1,3 +1,4 @@
+solbench/
 ignore/
 auto_evals/
 venv/
@@ -6,9 +7,4 @@ __pycache__/
 .ipynb_checkpoints
 *ipynb
 .vscode/
-
-eval-queue/
-eval-results/
-eval-queue-bk/
-eval-results-bk/
 logs/

README.md

@@ -40,10 +40,6 @@ Results files should have the following format and be stored as json files:
 }
 ```
 
-Request files are created automatically by this tool.
-
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
-
 # Code logic for more complex edits
 
 You'll find 

app.py

@@ -2,6 +2,8 @@
 # -*- coding: utf-8 -*-
 # flake8: noqa E501
 
+import shutil
+
 import gradio as gr
 from apscheduler.schedulers.background import BackgroundScheduler
 from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
@@ -10,7 +12,7 @@ from huggingface_hub import snapshot_download
 from src.about import (
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
+    EVALUATION_REQUESTS_TEXT,
     EVALUATION_SCRIPT,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
@@ -28,8 +30,17 @@ from src.display.utils import (
     WeightType,
     fields,
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.envs import (
+    API,
+    CACHE_PATH,
+    EVAL_REQUESTS_PATH,
+    EVAL_RESULTS_PATH,
+    REPO_ID,
+    REQUESTS_REPO,
+    RESULTS_REPO,
+    TOKEN,
+)
+from src.populate import get_evaluation_requests_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 
 
@@ -37,9 +48,10 @@ def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 # Space initialisation
+shutil.rmtree(CACHE_PATH, ignore_errors=True)
 try:
     snapshot_download(
-        repo_id=QUEUE_REPO,
+        repo_id=REQUESTS_REPO,
         local_dir=EVAL_REQUESTS_PATH,
         repo_type="dataset",
         tqdm_class=None,
@@ -48,6 +60,7 @@ try:
     )
 except Exception:
     restart_space()
+
 try:
     snapshot_download(
         repo_id=RESULTS_REPO,
@@ -60,7 +73,6 @@ try:
 except Exception:
     restart_space()
 
-
 LEADERBOARD_DF = get_leaderboard_df(
     EVAL_RESULTS_PATH,
     EVAL_REQUESTS_PATH,
@@ -69,10 +81,10 @@ LEADERBOARD_DF = get_leaderboard_df(
 )
 
 (
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+    finished_eval_requests_df,
+    running_eval_requests_df,
+    pending_eval_requests_df,
+) = get_evaluation_requests_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
@@ -124,85 +136,72 @@ with demo:
         with gr.TabItem("🧪 Submissions", elem_id="llm-benchmark-tab-table", id=3):
             with gr.Column():
                 with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+                    gr.Markdown(EVALUATION_REQUESTS_TEXT, elem_classes="markdown-text")
 
                 with gr.Column():
                     with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                        f"✅ Finished ({len(finished_eval_requests_df)})",
                         open=False,
                     ):
                         with gr.Row():
                             finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
+                                value=finished_eval_requests_df,
                                 headers=EVAL_COLS,
                                 datatype=EVAL_TYPES,
                                 row_count=5,
                             )
 
                     with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                        f"⏳ Pending ({len(pending_eval_requests_df)})",
                         open=False,
                     ):
                         with gr.Row():
                             pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
+                                value=pending_eval_requests_df,
                                 headers=EVAL_COLS,
                                 datatype=EVAL_TYPES,
                                 row_count=5,
                             )
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+                gr.Markdown("# ✉️ Submit a model", elem_classes="markdown-text")
 
             with gr.Row():
                 with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                     model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                        choices=[t.to_str(" ") for t in ModelType if t != ModelType.Unknown],
                         label="Model type",
                         multiselect=False,
                         value=None,
                         interactive=True,
                     )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
+                    # precision = gr.Dropdown(
+                    #     choices=[i.value.name for i in Precision if i != Precision.Unknown],
+                    #     label="Precision",
+                    #     multiselect=False,
+                    #     value="float16",
+                    #     interactive=True,
+                    # )
+                    # weight_type = gr.Dropdown(
+                    #     choices=[i.value.name for i in WeightType],
+                    #     label="Weights type",
+                    #     multiselect=False,
+                    #     value="Original",
+                    #     interactive=True,
+                    # )
+                    # base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+
+            submit_button = gr.Button("Submit")
             submission_result = gr.Markdown()
             submit_button.click(
                 add_new_eval,
                 [
                     model_name_textbox,
-                    base_model_name_textbox,
+                    # base_model_name_textbox,
                     revision_name_textbox,
-                    precision,
-                    weight_type,
+                    # precision,
+                    # weight_type,
                     model_type,
                 ],
                 submission_result,

src/about.py

@@ -16,19 +16,18 @@ class Task:
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("hf", "score", "LLM JUDGE")
-    task1 = Task("rouge1", "score", "ROUGE-unigrams")
-    task2 = Task("rouge2", "score", "ROUGE-bigrams")
-    task3 = Task("rougeL", "score", "ROUGE-Longest Common Subsequence")
-    task4 = Task("rougeLsum", "score", "ROUGE-Lsum")
-    task5 = Task("bleu", "score", "Bleu")
-    task6 = Task("brevity_penalty", "score", "Brevity Penalty")
-    task7 = Task("hf", "score", "LLM JUDGE")
-    # task = Task("precisions", "score", "Precision")
-
-
-NUM_FEWSHOT = 0  # Change with your few shot
+    # task_key, metric_key, title
+    task00 = Task("human_eval_solidity", "score", "HumanEval for Solidity")
+    # task01 = Task("naive_judge", "score", "NaïveJudge Average")
+    # task02 = Task("naive_judge_expert", "score", "NaïveJudge Expert")
+    # task03 = Task("naive_judge_average", "score", "NaïveJudge Average")
+    # task04 = Task("naive_judge_beginner", "score", "NaïveJudge Beginner")
+    task05 = Task("rouge1", "score", "ROUGE-unigrams")
+    task06 = Task("rouge2", "score", "ROUGE-bigrams")
+    task07 = Task("rougeL", "score", "ROUGE-Longest Common Subsequence")
+    task08 = Task("rougeLsum", "score", "ROUGE-Lsum")
+    task09 = Task("bleu", "score", "Bleu")
+    task10 = Task("brevity_penalty", "score", "Brevity Penalty")
 # ---------------------------------------------------
 
 # Your leaderboard name
@@ -47,7 +46,7 @@ To reproduce our results, here is the commands you can run:
 
 """
 
-EVALUATION_QUEUE_TEXT = """
+EVALUATION_REQUESTS_TEXT = """
 ## Some good practices before submitting a model
 
 ### 1) Make sure you can load your model and tokenizer using AutoClasses:
@@ -59,22 +58,10 @@ tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
 ```
 If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
 
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+Note: make sure your model is public.
 
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
+### 2) Fill up your model card
 When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
 """
 EVALUATION_SCRIPT = ''
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"

src/display/utils.py

@@ -47,10 +47,10 @@ model_info_columns = [
     ("weight_type", "Weight type", "str", False, True),
     ("precision", "Precision", "str", False),
     ("license", "License", "str", False),
-    ("params", "Parameters ⚙️", "number", False),
-    ("likes", "Likes ❤️", "number", False),
-    ("still_on_hub", "Available on the hub", "bool", False),
-    ("revision", "Model sha", "str", False, False),
+    ("params", "Parameters", "number", False),
+    ("likes", "Likes", "number", False),
+    ("still_on_hub", "Available on HuggingFace", "bool", False),
+    ("revision", "Revision", "str", False, False),
 ]
 
 for col_name, display_name, col_type, displayed_by_default, *args in model_info_columns:
@@ -60,15 +60,15 @@ for col_name, display_name, col_type, displayed_by_default, *args in model_info_
 # Create the AutoEvalColumn dataclass
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)()
 
-# For the queue columns in the submission tab
+# For the requests columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
+    # private = ColumnContent("private", "bool", True)
+    # precision = ColumnContent("precision", "str", True)
+    # weight_type = ColumnContent("weight_type", "str", "Original")
+    # status = ColumnContent("status", "str", True)
 
 
 # All the model information that we might need
@@ -80,26 +80,21 @@ class ModelDetails:
 
 
 class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    CS = ModelDetails(name="closed-source", symbol="🔒")
+    PT = ModelDetails(name="pretrained", symbol="💎")
+    FT = ModelDetails(name="finetuned", symbol="💍")
     BrainDAO = ModelDetails(name="braindao", symbol="🧠")
     Unknown = ModelDetails(name="", symbol="❓")
 
-    def to_str(self, separator=" "):
+    def to_str(self, separator="  "):
         return f"{self.value.symbol}{separator}{self.value.name}"
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
+        if "finetuned" in type or "💍" in type:
             return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
+        if "pretrained" in type or "💎" in type:
             return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
+        if "braindao" in type or "🧠" in type:
             return ModelType.IFT
         return ModelType.Unknown
 
@@ -113,13 +108,19 @@ class WeightType(Enum):
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
+    float32 = ModelDetails("float32")
+    bfloat32 = ModelDetails("bfloat32")
+    Unknown = ModelDetails("Unknown")
 
     def from_str(precision):
         if precision in ["torch.float16", "float16"]:
             return Precision.float16
         if precision in ["torch.bfloat16", "bfloat16"]:
             return Precision.bfloat16
+        if precision in ["torch.float32", "float32"]:
+            return Precision.float32
+        if precision in ["torch.bfloat32", "bfloat32"]:
+            return Precision.bfloat32
         return Precision.Unknown
 
 

src/envs.py

@@ -8,22 +8,20 @@ from huggingface_hub import HfApi
 
 # Info to change for your repository
 # ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
+TOKEN = os.environ.get("HF_TOKEN")
 
-OWNER = "braindao"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "braindao"
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/solidity-leaderboard"
-QUEUE_REPO = f"{OWNER}/solbench-leaderboard-queue"
+REQUESTS_REPO = f"{OWNER}/solbench-leaderboard-requests"
 RESULTS_REPO = f"{OWNER}/solbench-leaderboard-results"
 
 # If you setup a cache later, just change HF_HOME
-CACHE_PATH = os.getenv("HF_HOME", ".")
+CACHE_PATH = os.path.join(os.getenv("HF_HOME", "."), "solbench")
 
 # Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "requests")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "results")
 
 API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -13,6 +13,7 @@ import numpy as np
 from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, ModelType, Precision, Tasks, WeightType
 from src.submission.check_validity import is_model_on_hub
+from src.utils import get_model_name, get_org_and_model_names, get_request_hash
 
 
 @dataclass
@@ -20,7 +21,7 @@ class EvalResult:
     """Represents one full evaluation. Built from a combination of the result and request file for a given run.
     """
     eval_name: str  # org_model_precision (uid)
-    full_model: str  # org/model (path on hub)
+    model_name: str  # org/model (path on hub)
     org: str
     model: str
     revision: str  # commit hash, "" if main
@@ -29,7 +30,7 @@ class EvalResult:
     model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
     weight_type: WeightType = WeightType.Original  # Original or Adapter
     architecture: str = "Unknown"
-    license: str = "?"
+    license: str = "Unknown"
     likes: int = 0
     num_params: int = 0
     date: str = ""  # submission date of request file
@@ -41,32 +42,22 @@ class EvalResult:
         with open(json_filepath) as fp:
             data = json.load(fp)
 
+        org, model = get_org_and_model_names(json_filepath)
         config = data.get("config")
 
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
 
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
+        result_key = f"{org}_{model}_{precision.value.name}"
+        model_name = get_model_name(json_filepath)
 
         still_on_hub, _, model_config = is_model_on_hub(
-            full_model,
+            model_name,
             config.get("model_sha", "main"),
             trust_remote_code=True,
             test_tokenizer=False,
         )
-        architecture = "?"
+        architecture = "Unknown"
         if model_config is not None:
             architectures = getattr(model_config, "architectures", None)
             if architectures:
@@ -89,7 +80,7 @@ class EvalResult:
 
         return cls(
             eval_name=result_key,
-            full_model=full_model,
+            model_name=model_name,
             org=org,
             model=model,
             results=results,
@@ -101,14 +92,19 @@ class EvalResult:
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
+        request_file = get_request_file_for_model(
+            requests_path,
+            self.model_name,
+            self.revision,
+            self.precision.value.name,
+        )
+        
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
             self.model_type = ModelType.from_str(request.get("model_type", ""))
             self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
+            self.license = request.get("license", "Unknown")
             self.likes = request.get("likes", 0)
             self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
@@ -125,7 +121,7 @@ class EvalResult:
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.model.name: make_clickable_model(self.model_name),
             AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
             AutoEvalColumn.license.name: self.license,
@@ -140,26 +136,17 @@ class EvalResult:
         return data_dict
 
 
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
+def get_request_file_for_model(
+        requests_path: str,
+        model_name: str,
+        revision: str,
+        precision: str,
+    ):
+    request_hash = get_request_hash(model_name, revision, precision)
+    filepath = os.path.join(requests_path, model_name, '{}.json'.format(request_hash))
+    print(f'Loading {filepath}...')
+    filepath = glob.glob(filepath)[0]
+    return filepath
 
 
 def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:

src/populate.py

@@ -10,6 +10,7 @@ import pandas as pd
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
+from src.utils import get_model_name
 
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
@@ -26,48 +27,91 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     return df
 
 
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+# def get_evaluation_requests_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+#     """Creates the different dataframes for the evaluation requestss requestes"""
+#     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+#     all_evals = []
+
+#     for entry in entries:
+#         if ".json" in entry:
+#             file_path = os.path.join(save_path, entry)
+#             try:
+#                 with open(file_path, encoding='utf-8') as fp:
+#                     data = json.load(fp)
+#             except UnicodeDecodeError as e:
+#                 print(f"Unicode decoding error in {file_path}: {e}")
+#                 continue
+
+#             # data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+#             model_name = get_model_name(file_path)
+#             data[EvalQueueColumn.model.name] = make_clickable_model(model_name)
+
+#             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+#             all_evals.append(data)
+
+#         elif ".md" not in entry:
+#             # this is a folder
+#             sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+#             for sub_entry in sub_entries:
+#                 file_path = os.path.join(save_path, entry, sub_entry)
+#                 try:
+#                     with open(file_path, encoding='utf-8') as fp:
+#                         data = json.load(fp)
+#                 except json.JSONDecodeError:
+#                     print(f"Error reading {file_path}")
+#                     continue
+#                 except UnicodeDecodeError as e:
+#                     print(f"Unicode decoding error in {file_path}: {e}")
+#                     continue
+
+#                 # data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+#                 model_name = get_model_name(file_path)
+#                 data[EvalQueueColumn.model.name] = make_clickable_model(model_name)
+
+#                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+#                 all_evals.append(data)
+
+#     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+#     running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+#     finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+#     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+#     df_running = pd.DataFrame.from_records(running_list, columns=cols)
+#     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+#     return df_finished[cols], df_running[cols], df_pending[cols]
+
+def get_evaluation_requests_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    """Creates the different dataframes for the evaluation requestss requested."""
     all_evals = []
 
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            try:
-                with open(file_path, encoding='utf-8') as fp:
-                    data = json.load(fp)
-            except UnicodeDecodeError as e:
-                print(f"Unicode decoding error in {file_path}: {e}")
-                continue
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                try:
-                    with open(file_path, encoding='utf-8') as fp:
-                        data = json.load(fp)
-                except json.JSONDecodeError:
-                    print(f"Error reading {file_path}")
-                    continue
-                except UnicodeDecodeError as e:
-                    print(f"Unicode decoding error in {file_path}: {e}")
-                    continue
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
+    def process_file(file_path):
+        try:
+            with open(file_path, 'r', encoding='utf-8') as fp:
+                data = json.load(fp)
+        except (json.JSONDecodeError, UnicodeDecodeError) as e:
+            print(f"Error reading or decoding {file_path}: {e}")
+            return None
+
+        model_name = get_model_name(file_path)
+        # data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+        data[EvalQueueColumn.model.name] = make_clickable_model(model_name)
+        data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+        return data
+
+    for root, _, files in os.walk(save_path):
+        for file in files:
+            if file.endswith('.json'):
+                file_path = os.path.join(root, file)
+                data = process_file(file_path)
+                if data:
+                    all_evals.append(data)
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
     running_list = [e for e in all_evals if e["status"] == "RUNNING"]
     finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+
     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
     df_running = pd.DataFrame.from_records(running_list, columns=cols)
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+
     return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -63,7 +63,7 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
         return False, "was not found on hub!", None
 
 
-def get_model_size(model_info: ModelInfo, precision: str):
+def get_model_size(model_info: ModelInfo, precision: str = None):
     """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
     try:
         model_size = round(model_info.safetensors["total"] / 1e9, 3)

src/submission/submit.py

@@ -7,8 +7,9 @@ import os
 from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO, TOKEN
+from src.envs import API, EVAL_REQUESTS_PATH, REQUESTS_REPO, TOKEN
 from src.submission.check_validity import already_submitted_models, check_model_card, get_model_size, is_model_on_hub
+from src.utils import get_request_hash, get_request_id
 
 REQUESTED_MODELS = None
 USERS_TO_SUBMISSION_DATES = None
@@ -16,12 +17,16 @@ USERS_TO_SUBMISSION_DATES = None
 
 def add_new_eval(
     model: str,
-    base_model: str,
+    # base_model: str,
     revision: str,
-    precision: str,
-    weight_type: str,
+    # precision: str,
+    # weight_type: str,
     model_type: str,
 ):
+    # Remove if added to the form
+    weight_type = 'Original'
+    precision = 'Unknown'
+
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
@@ -44,12 +49,12 @@ def add_new_eval(
         revision = "main"
 
     # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
+    # if weight_type in ["Delta", "Adapter"]:
+    #     base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+    #     if not base_model_on_hub:
+    #         return styled_error(f'Base model "{base_model}" {error}')
 
-    if not weight_type == "Adapter":
+    if weight_type != "Adapter":
         model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')
@@ -60,7 +65,10 @@ def add_new_eval(
     except Exception:
         return styled_error("Could not get your model information. Please fill it up properly.")
 
-    model_size = get_model_size(model_info=model_info, precision=precision)
+    model_size = get_model_size(
+        model_info=model_info,
+        precision=precision,
+    )
 
     # Were the model card and license filled?
     try:
@@ -76,8 +84,8 @@ def add_new_eval(
     print("Adding new eval")
 
     eval_entry = {
-        "model": model,
-        "base_model": base_model,
+        # "model": model,
+        # "base_model": base_model,
         "revision": revision,
         "precision": precision,
         "weight_type": weight_type,
@@ -91,13 +99,15 @@ def add_new_eval(
     }
 
     # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+    request_id = get_request_id(model, revision, precision)
+    if request_id in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted.")
+    request_hash = get_request_hash(model, revision, precision)
 
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    out_path = f"{OUT_DIR}/{model_path}/{request_hash}.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))
@@ -105,15 +115,15 @@ def add_new_eval(
     print("Uploading eval file")
     API.upload_file(
         path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
+        path_in_repo=out_path.split("eval-requests/")[1],
+        repo_id=REQUESTS_REPO,
         repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
+        commit_message=f"Add {model} to eval requests",
     )
 
     # Remove the local file
     os.remove(out_path)
 
     return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+        "Your request has been submitted to the evaluation requests!\nPlease wait for up to an hour for the model to show in the PENDING list."
     )

src/utils.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import hashlib
+
+
+def get_org_and_model_names(file_path: str) -> str:
+    org, model = file_path.split("/")[-3:-1]
+    model = model.removesuffix(".json")
+    model = model.split('_request_')[0]
+    return org, model
+
+
+def get_model_name(file_path: str) -> str:
+    org, model = get_org_and_model_names(file_path)
+    return f"{org}/{model}"
+
+
+def get_hash(key: str) -> str:
+    sha256_hash = hashlib.sha256()
+    sha256_hash.update(key.encode('utf-8'))
+    return sha256_hash.hexdigest()[:16]
+
+
+def get_request_id(model: str, revision: str, precision: str) -> str:
+    return f"{model}_{revision}_{precision}"
+
+
+def get_request_hash(model: str, revision: str, precision: str) -> str:
+    return get_hash(get_request_id(model, revision, precision))