Fix leaderboard errors

src/display/utils.py

@@ -1,5 +1,6 @@
-from dataclasses import dataclass, make_dataclass
+from dataclasses import dataclass, make_dataclass, field
 from enum import Enum
+from typing import List
 
 import pandas as pd
 
@@ -8,7 +9,6 @@ from src.about import Tasks
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
-
 @dataclass
 class ColumnContent:
     name: str
@@ -18,27 +18,41 @@ class ColumnContent:
     never_hidden: bool = False
 
 ## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Security Score ⬆️", "number", True)])
+def create_column_field(name: str, type: str, displayed_by_default: bool, hidden: bool = False, never_hidden: bool = False):
+    return field(default_factory=lambda: ColumnContent(name, type, displayed_by_default, hidden, never_hidden))
+
+auto_eval_column_dict = [
+    # Init
+    ("model_type_symbol", ColumnContent, create_column_field("T", "str", True, never_hidden=True)),
+    ("model", ColumnContent, create_column_field("Model", "markdown", True, never_hidden=True)),
+    # Scores
+    ("average", ColumnContent, create_column_field("Security Score ⬆️", "number", True))
+]
+
+# Add task-specific columns
 for task in Tasks:
     if task.name == "safetensors":
-        auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent("Safetensors ✓", "bool", True)])
+        auto_eval_column_dict.append((task.name, ColumnContent, create_column_field("Safetensors ✓", "bool", True)))
     else:
-        auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+        auto_eval_column_dict.append((task.name, ColumnContent, create_column_field(task.value.col_name, "number", True)))
+
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight Format", "str", True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", True)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on Hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model SHA", "str", False, False)])
+model_info_columns = [
+    ("model_type", "Type", "str", False),
+    ("architecture", "Architecture", "str", False),
+    ("weight_type", "Weight Format", "str", True),
+    ("precision", "Precision", "str", False),
+    ("license", "Hub License", "str", True),
+    ("params", "#Params (B)", "number", False),
+    ("likes", "Hub ❤️", "number", False),
+    ("still_on_hub", "Available on Hub", "bool", False),
+    ("revision", "Model SHA", "str", False)
+]
+
+for field_name, display_name, field_type, displayed_by_default in model_info_columns:
+    auto_eval_column_dict.append(
+        (field_name, ColumnContent, create_column_field(display_name, field_type, displayed_by_default))
+    )
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -46,12 +60,12 @@ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", True)
-    status = ColumnContent("status", "str", True)
+    model: ColumnContent = field(default_factory=lambda: ColumnContent("model", "markdown", True))
+    revision: ColumnContent = field(default_factory=lambda: ColumnContent("revision", "str", True))
+    private: ColumnContent = field(default_factory=lambda: ColumnContent("private", "bool", True))
+    precision: ColumnContent = field(default_factory=lambda: ColumnContent("precision", "str", True))
+    weight_type: ColumnContent = field(default_factory=lambda: ColumnContent("weight_type", "str", True))
+    status: ColumnContent = field(default_factory=lambda: ColumnContent("status", "str", True))
 
 ## All the model information that we might need
 @dataclass
@@ -60,7 +74,6 @@ class ModelDetails:
     display_name: str = ""
     symbol: str = "" # emoji
 
-
 class ModelType(Enum):
     PT = ModelDetails(name="pretrained", symbol="🟢")
     FT = ModelDetails(name="fine-tuned", symbol="🔶")
@@ -93,6 +106,7 @@ class Precision(Enum):
     bfloat16 = ModelDetails("bfloat16")
     Unknown = ModelDetails("?")
 
+    @staticmethod
     def from_str(precision):
         if precision in ["torch.float16", "float16"]:
             return Precision.float16

src/envs.py

@@ -7,10 +7,10 @@ from huggingface_hub import HfApi
 TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
 
 # Change these to your organization name
-OWNER = "security-llm-leaderboard"  # Create this organization on HuggingFace
+OWNER = "stacklok"  # Create this organization on HuggingFace
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
+REPO_ID = f"{OWNER}/secure-llm-leaderboard"
 QUEUE_REPO = f"{OWNER}/requests"
 RESULTS_REPO = f"{OWNER}/results"
 

src/submission/check_validity.py

@@ -1,8 +1,6 @@
 import json
 import os
-import re
 from collections import defaultdict
-from datetime import datetime, timedelta, timezone
 
 import huggingface_hub
 from huggingface_hub import ModelCard