update

app.py

@@ -19,10 +19,10 @@ from src.display.utils import (
     EVAL_COLS,
     EVAL_TYPES,
     AutoEvalColumn,
-    ModelType,
+    LibraryType,
     fields,
-    WeightType,
-    Precision
+    Language,
+    AssessmentStatus
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
@@ -68,23 +68,23 @@ def init_leaderboard(dataframe):
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        search_columns=[AutoEvalColumn.library.name, AutoEvalColumn.license.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            ColumnFilter(AutoEvalColumn.library_type.name, type="checkboxgroup", label="Library types"),
+            ColumnFilter(AutoEvalColumn.language.name, type="checkboxgroup", label="Programming Language"),
             ColumnFilter(
-                AutoEvalColumn.params.name,
+                AutoEvalColumn.stars.name,
                 type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
+                min=0,
+                max=50000,
+                label="GitHub Stars",
             ),
             ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+                AutoEvalColumn.availability.name, type="boolean", label="Show only active libraries", default=True
             ),
         ],
-        bool_checkboxgroup_label="Hide models",
+        bool_checkboxgroup_label="Filter libraries",
         interactive=False,
     )
 
@@ -95,20 +95,20 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅 Vulnerability Leaderboard", elem_id="vulnerability-leaderboard-tab", id=0):
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+        with gr.TabItem("📝 About", elem_id="about-tab", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+        with gr.TabItem("🚀 Submit Library", elem_id="submit-library-tab", id=3):
             with gr.Column():
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
                 with gr.Column():
                     with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                        f"✅ Completed Assessments ({len(finished_eval_queue_df)})",
                         open=False,
                     ):
                         with gr.Row():
@@ -119,7 +119,7 @@ with demo:
                                 row_count=5,
                             )
                     with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                        f"🔄 In Progress Assessments ({len(running_eval_queue_df)})",
                         open=False,
                     ):
                         with gr.Row():
@@ -131,7 +131,7 @@ with demo:
                             )
 
                     with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                        f"⏳ Pending Assessment Queue ({len(pending_eval_queue_df)})",
                         open=False,
                     ):
                         with gr.Row():
@@ -142,48 +142,42 @@ with demo:
                                 row_count=5,
                             )
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+                gr.Markdown("# ✉️✨ Submit a library for vulnerability assessment", elem_classes="markdown-text")
 
             with gr.Row():
                 with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
+                    library_name_textbox = gr.Textbox(label="Library name (org/repo format)")
+                    library_version_textbox = gr.Textbox(label="Version", placeholder="v1.0.0")
+                    library_type = gr.Dropdown(
+                        choices=[t.to_str(" : ") for t in LibraryType if t != LibraryType.Unknown],
+                        label="Library type",
                         multiselect=False,
                         value=None,
                         interactive=True,
                     )
 
                 with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
+                    language = gr.Dropdown(
+                        choices=[i.value.name for i in Language if i != Language.Other],
+                        label="Programming Language",
                         multiselect=False,
-                        value="float16",
+                        value="Python",
                         interactive=True,
                     )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+                    framework = gr.Textbox(label="Framework/Ecosystem (e.g., PyTorch, React)")
+                    repository_url = gr.Textbox(label="Repository URL")
 
-            submit_button = gr.Button("Submit Eval")
+            submit_button = gr.Button("Submit for Assessment")
             submission_result = gr.Markdown()
             submit_button.click(
                 add_new_eval,
                 [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
+                    library_name_textbox,
+                    library_version_textbox,
+                    repository_url,
+                    language,
+                    framework,
+                    library_type,
                 ],
                 submission_result,
             )

src/about.py

@@ -11,62 +11,96 @@ class Task:
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-NUM_FEWSHOT = 0 # Change with your few shot
+    # Risk domains from LibVulnWatch paper
+    license = Task("license_validation", "score", "License Risk")
+    security = Task("security_assessment", "score", "Security Risk")
+    maintenance = Task("maintenance_health", "score", "Maintenance Risk") 
+    dependency = Task("dependency_management", "score", "Dependency Risk")
+    regulatory = Task("regulatory_compliance", "score", "Regulatory Risk")
+
+NUM_FEWSHOT = 0 # Not relevant for vulnerability assessment
 # ---------------------------------------------------
 
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">LibVulnWatch: Vulnerability Assessment Leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+## Systematic Vulnerability Assessment and Leaderboard Tracking for Open-Source AI Libraries
+
+This leaderboard provides continuous vulnerability assessment for open-source AI libraries across five critical risk domains:
+- **License Validation**: Legal risks based on license type, compatibility, and requirements
+- **Security Assessment**: Vulnerability severity and patch responsiveness
+- **Maintenance Health**: Sustainability and governance practices
+- **Dependency Management**: Vulnerability inheritance and supply chain security
+- **Regulatory Compliance**: Compliance readiness for various frameworks
+
+Lower scores indicate fewer vulnerabilities and lower risk. The overall risk score is a weighted average of all domains, with security given higher priority.
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-## How it works
+## How LibVulnWatch Works
 
-## Reproducibility
-To reproduce our results, here is the commands you can run:
+Our assessment methodology evaluates libraries through:
+1. **Static Analysis**: Code review, license parsing, and documentation examination
+2. **Dynamic Analysis**: Vulnerability scanning, dependency checking, and API testing  
+3. **Metadata Analysis**: Repository metrics, contributor patterns, and release cadence
 
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
+Each library receives a risk score (0-10) in each domain, with lower scores indicating lower risk.
 
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
+## Reproducibility
+To reproduce our assessment for a specific library:
 ```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+from libvulnwatch import VulnerabilityAssessor
 
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+# Initialize the assessor
+assessor = VulnerabilityAssessor()
 
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+# Run assessment on a library
+results = assessor.assess_library("organization/library_name")
 
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
+# View detailed results
+print(results.risk_scores)
+print(results.detailed_findings)
+```
+"""
 
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+EVALUATION_QUEUE_TEXT = """
+## Before submitting a library for assessment
+
+### 1) Ensure your library is publicly accessible
+LibVulnWatch can only assess libraries that are publicly available on GitHub or another accessible repository.
+
+### 2) Verify complete metadata is available
+Our assessment relies on metadata including:
+- License information
+- Dependency specifications
+- Maintenance history and contributor information
+- Security policies and vulnerability handling processes
+
+### 3) Make sure your repository has an open license
+This leaderboard is designed for open-source AI libraries, which should have clear licensing terms.
+
+### 4) Add security documentation
+Libraries with comprehensive security documentation tend to receive better assessments.
+
+## If your assessment fails
+If your library shows as "FAILED" in the assessment queue, check that:
+- The repository is publicly accessible
+- All required metadata files are present 
+- Dependencies can be resolved
+- The repository doesn't employ obfuscation techniques that interfere with analysis
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
+@article{LibVulnWatch2025,
+  title={LibVulnWatch: Systematic Vulnerability Assessment and Leaderboard Tracking for Open-Source AI Libraries},
+  author={First Author and Second Author},
+  journal={ICML 2025 Technical AI Governance Workshop},
+  year={2025}
+}
 """

src/display/formatting.py

@@ -1,3 +1,7 @@
+"""Helper functions to style our gradio elements"""
+
+import re
+
 def model_hyperlink(link, model_name):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
@@ -25,3 +29,44 @@ def has_no_nan_values(df, columns):
 
 def has_nan_values(df, columns):
     return df[columns].isna().any(axis=1)
+
+
+def make_clickable_library(library_name: str) -> str:
+    """Link to the GitHub repository"""
+    library_path = library_name.replace(" ", "-").lower()
+    
+    # If this is a GitHub repository, link directly
+    github_url = f"https://github.com/{library_path}"
+    
+    return f'<a href="{github_url}" target="_blank">{library_name}</a>'
+
+
+def styled_message(message) -> str:
+    """Format a message with a green header"""
+    return f'<span style="color: green">✅ Success:</span> {message}'
+
+
+def styled_warning(message) -> str:
+    """Format a warning message with an orange header"""
+    return f'<span style="color: orange">⚠️ Warning:</span> {message}'
+
+
+def styled_error(message) -> str:
+    """Format an error message with a red header"""
+    return f'<span style="color: red">❌ Error:</span> {message}'
+
+
+# Risk severity coloring for risk scores
+def colorize_risk_score(score):
+    """
+    Apply color coding to risk scores:
+    0-3.9: Green (Low risk)
+    4-6.9: Orange (Medium risk)
+    7-10: Red (High risk)
+    """
+    if score < 4:
+        return f'<span style="color: green">{score:.1f}</span>'
+    elif score < 7:
+        return f'<span style="color: orange">{score:.1f}</span>'
+    else:
+        return f'<span style="color: red">{score:.1f}</span>'

src/display/utils.py

@@ -23,22 +23,22 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["library_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["library", ColumnContent, ColumnContent("Library", "markdown", True, never_hidden=True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["overall_risk", ColumnContent, ColumnContent("Overall Risk ⬇️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+# Library information
+auto_eval_column_dict.append(["library_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["framework", ColumnContent, ColumnContent("Framework", "str", False)])
+auto_eval_column_dict.append(["version", ColumnContent, ColumnContent("Version", "str", False, False)])
+auto_eval_column_dict.append(["language", ColumnContent, ColumnContent("Language", "str", False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("License", "str", True)])
+auto_eval_column_dict.append(["stars", ColumnContent, ColumnContent("GitHub ⭐", "number", False)])
+auto_eval_column_dict.append(["last_update", ColumnContent, ColumnContent("Last Updated", "str", False)])
+auto_eval_column_dict.append(["verified", ColumnContent, ColumnContent("Independently Verified", "bool", False)])
+auto_eval_column_dict.append(["availability", ColumnContent, ColumnContent("Active Maintenance", "bool", True)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -46,59 +46,58 @@ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
+    library = ColumnContent("library", "markdown", True)
+    version = ColumnContent("version", "str", True)
+    language = ColumnContent("language", "str", True) 
+    framework = ColumnContent("framework", "str", True)
+    library_type = ColumnContent("library_type", "str", True)
     status = ColumnContent("status", "str", True)
 
-## All the model information that we might need
+## All the library information that we might need
 @dataclass
-class ModelDetails:
+class LibraryDetails:
     name: str
     display_name: str = ""
     symbol: str = "" # emoji
 
 
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
+class LibraryType(Enum):
+    ML = LibraryDetails(name="machine learning", symbol="🟢")
+    LLM = LibraryDetails(name="llm framework", symbol="🔶")
+    AGENT = LibraryDetails(name="agent framework", symbol="⭕")
+    VIS = LibraryDetails(name="visualization", symbol="🟦")
+    GENERAL = LibraryDetails(name="general ai", symbol="🟣")
+    Unknown = LibraryDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
         return f"{self.value.symbol}{separator}{self.value.name}"
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
+        if "machine learning" in type or "🟢" in type:
+            return LibraryType.ML
+        if "llm framework" in type or "🔶" in type:
+            return LibraryType.LLM 
+        if "agent framework" in type or "⭕" in type:
+            return LibraryType.AGENT
+        if "visualization" in type or "🟦" in type:
+            return LibraryType.VIS
+        if "general ai" in type or "🟣" in type:
+            return LibraryType.GENERAL
+        return LibraryType.Unknown
+
+class Language(Enum):
+    Python = LibraryDetails("Python")
+    JavaScript = LibraryDetails("JavaScript")
+    TypeScript = LibraryDetails("TypeScript") 
+    Java = LibraryDetails("Java")
+    CPP = LibraryDetails("C++")
+    Other = LibraryDetails("Other")
+
+class AssessmentStatus(Enum):
+    Verified = LibraryDetails("Verified")
+    Unverified = LibraryDetails("Unverified")
+    Disputed = LibraryDetails("Disputed")
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]

src/envs.py

@@ -6,20 +6,20 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "libvulnwatch" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+QUEUE_REPO = f"{OWNER}/vulnerability-requests"
+RESULTS_REPO = f"{OWNER}/vulnerability-assessments"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")
 
 # Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "assessment-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "assessment-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "assessment-queue-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "assessment-results-bk")
 
 API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -3,144 +3,156 @@ import json
 import math
 import os
 from dataclasses import dataclass
+from datetime import datetime
 
-import dateutil
 import numpy as np
 
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
+from src.display.formatting import make_clickable_library
+from src.display.utils import AutoEvalColumn, LibraryType, Tasks, Language, AssessmentStatus
 
 
 @dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+class AssessmentResult:
+    """Represents one full vulnerability assessment. Built from a combination of the result and request file for a given library.
     """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
+    assessment_id: str # Unique identifier
+    library_name: str # org/repo
     org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
+    repo: str
+    version: str
+    results: dict # Risk scores
+    framework: str = ""
+    language: Language = Language.Other
+    library_type: LibraryType = LibraryType.Unknown
     license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
+    stars: int = 0
+    last_update: str = ""
+    availability: bool = True
+    verified: bool = False
 
     @classmethod
     def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
+        """Initializes the assessment result from a JSON file"""
         with open(json_filepath) as fp:
             data = json.load(fp)
 
-        config = data.get("config")
+        assessment = data.get("assessment", {})
+        # Get library and org
+        library_name = assessment.get("library_name", "")
+        org_and_repo = library_name.split("/", 1)
 
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
+        if len(org_and_repo) == 1:
             org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
+            repo = org_and_repo[0]
+            assessment_id = f"{repo}_{assessment.get('version', '')}"
         else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
+            org = org_and_repo[0]
+            repo = org_and_repo[1]
+            assessment_id = f"{org}_{repo}_{assessment.get('version', '')}"
 
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
+        # Extract risk scores
+        risk_scores = {}
         for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
+            domain = task.value
+            score = assessment.get("scores", {}).get(domain.benchmark, None)
+            if score is not None:
+                risk_scores[domain.benchmark] = score
+
+        # Library metadata
+        framework = assessment.get("framework", "")
+        language_str = assessment.get("language", "Other")
+        language = next((lang for lang in Language if lang.value.name == language_str), Language.Other)
+        
+        # Availability and verification
+        last_update = assessment.get("last_updated", "")
+        if last_update:
+            try:
+                # Format date for display
+                dt = datetime.fromisoformat(last_update)
+                last_update = dt.strftime("%Y-%m-%d")
+            except:
+                pass
+                
         return self(
-            eval_name=result_key,
-            full_model=full_model,
+            assessment_id=assessment_id,
+            library_name=library_name,
             org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
+            repo=repo,
+            version=assessment.get("version", ""),
+            results=risk_scores,
+            framework=framework,
+            language=language,
+            license=assessment.get("license", "?"),
+            availability=assessment.get("active_maintenance", True),
+            verified=assessment.get("independently_verified", False),
+            last_update=last_update,
         )
 
     def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+        """Finds the relevant request file for the current library and updates info with it"""
+        request_file = get_request_file_for_library(requests_path, self.library_name, self.version)
 
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
+            self.library_type = LibraryType.from_str(request.get("library_type", ""))
+            self.stars = request.get("stars", 0)
         except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+            print(f"Could not find request file for {self.library_name} version {self.version}")
 
     def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        """Converts the Assessment Result to a dict compatible with our dataframe display"""
+        # Calculate overall risk as weighted average
+        weights = {
+            "license_validation": 0.2,
+            "security_assessment": 0.3,
+            "maintenance_health": 0.2,
+            "dependency_management": 0.2,
+            "regulatory_compliance": 0.1
+        }
+        
+        # Calculate overall risk - if domain is missing, use highest risk score (10)
+        risk_sum = 0
+        weight_sum = 0
+        
+        for domain, weight in weights.items():
+            score = self.results.get(domain, 10)  # Default to highest risk if missing
+            risk_sum += score * weight
+            weight_sum += weight
+            
+        overall_risk = risk_sum / weight_sum if weight_sum > 0 else 10
+            
         data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
+            "assessment_id": self.assessment_id,  # not a column, just a save name
+            AutoEvalColumn.library_type.name: self.library_type.value.name,
+            AutoEvalColumn.library_type_symbol.name: self.library_type.value.symbol,
+            AutoEvalColumn.language.name: self.language.value.name, 
+            AutoEvalColumn.framework.name: self.framework,
+            AutoEvalColumn.library.name: make_clickable_library(self.library_name),
+            AutoEvalColumn.version.name: self.version,
+            AutoEvalColumn.overall_risk.name: overall_risk,
             AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            AutoEvalColumn.stars.name: self.stars,
+            AutoEvalColumn.last_update.name: self.last_update,
+            AutoEvalColumn.verified.name: self.verified,
+            AutoEvalColumn.availability.name: self.availability,
         }
 
         for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+            data_dict[task.name] = self.results.get(task.value.benchmark, 10)  # Default to highest risk
 
         return data_dict
 
 
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+def get_request_file_for_library(requests_path, library_name, version):
+    """Selects the correct request file for a given library. Only keeps runs tagged as FINISHED"""
     request_files = os.path.join(
         requests_path,
-        f"{model_name}_eval_request_*.json",
+        f"{library_name.replace('/', '_')}_eval_request_*.json",
     )
     request_files = glob.glob(request_files)
 
-    # Select correct request file (precision)
+    # Select correct request file (version)
     request_file = ""
     request_files = sorted(request_files, reverse=True)
     for tmp_request_file in request_files:
@@ -148,45 +160,45 @@ def get_request_file_for_model(requests_path, model_name, precision):
             req_content = json.load(f)
             if (
                 req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
+                and req_content["version"] == version
             ):
                 request_file = tmp_request_file
     return request_file
 
 
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
+def get_raw_assessment_results(results_path: str, requests_path: str) -> list[AssessmentResult]:
+    """From the path of the results folder root, extract all needed info for assessments"""
+    assessment_filepaths = []
 
     for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
+        # We should only have json files in assessment results
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue
 
-        # Sort the files by date
+        # Sort the files by date if they have date info
         try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
+            files.sort(key=lambda x: datetime.fromisoformat(json.loads(open(os.path.join(root, x)).read())["assessment"]["completed_time"]), reverse=True)
+        except:
+            pass
 
         for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
+            assessment_filepaths.append(os.path.join(root, file))
 
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
+    assessment_results = {}
+    for assessment_filepath in assessment_filepaths:
         # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
+        assessment_result = AssessmentResult.init_from_json_file(assessment_filepath)
+        assessment_result.update_with_request_file(requests_path)
 
         # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        assessment_id = assessment_result.assessment_id
+        if assessment_id in assessment_results.keys():
+            assessment_results[assessment_id].results.update({k: v for k, v in assessment_result.results.items() if v is not None})
         else:
-            eval_results[eval_name] = eval_result
+            assessment_results[assessment_id] = assessment_result
 
     results = []
-    for v in eval_results.values():
+    for v in assessment_results.values():
         try:
             v.to_dict() # we test if the dict version is complete
             results.append(v)

src/populate.py

@@ -1,58 +1,100 @@
-import json
-import os
+"""Functions to populate the leaderboard"""
 
 import pandas as pd
 
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
+from src.display.utils import AutoEvalColumn
+from src.leaderboard.read_evals import get_raw_assessment_results
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
+def get_leaderboard_df(eval_results_path, eval_requests_path, cols, benchmark_cols):
+    """Read all the runs in the folder and return a dataframe
+    
+    Args:
+        eval_results_path: Path to the assessment result files
+        eval_requests_path: Path to the assessment request files
+        cols: Columns names to include in the dataframe
+        benchmark_cols: Risk categories column names
+        
+    Returns:
+        Pandas dataframe for the leaderboard
+    """
+    try:
+        assessment_results = get_raw_assessment_results(eval_results_path, eval_requests_path)
+        
+        # If we get results, convert to dataframe
+        if len(assessment_results) > 0:
+            # Create dataframe from assessment results
+            all_df = pd.DataFrame.from_records([r.to_dict() for r in assessment_results])
+            
+            # Sort by overall risk score (ascending - lower is better)
+            if AutoEvalColumn.overall_risk.name in all_df.columns:
+                all_df = all_df.sort_values(by=[AutoEvalColumn.overall_risk.name])
+                
+            return all_df
+            
+        return pd.DataFrame(columns=cols)  # Empty dataframe with columns
+    except Exception as e:
+        print(f"Error reading evaluation results: {e}")
+        return pd.DataFrame(columns=cols)  # Return empty dataframe
 
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]
+def get_evaluation_queue_df(eval_requests_path, eval_cols):
+    """Read from the evaluation queue directory and return dataframes for each status
+    
+    Args:
+        eval_requests_path: Path to the assessment request files
+        eval_cols: Columns for the queue dataframes
+        
+    Returns:
+        Tuple of dataframes (finished, running, pending)
+    """
+    try:
+        import glob
+        import json
+        import os
+        
+        # Find all request files
+        request_files = glob.glob(os.path.join(eval_requests_path, "*.json"))
+        
+        finished_data = []
+        running_data = []
+        pending_data = []
+        
+        for file_path in request_files:
+            try:
+                with open(file_path, "r") as f:
+                    data = json.load(f)
+                    
+                    # Extract relevant fields
+                    row = {
+                        "library": data.get("library", ""),
+                        "version": data.get("version", ""),
+                        "language": data.get("language", ""),
+                        "framework": data.get("framework", ""),
+                        "library_type": data.get("library_type", ""),
+                        "status": data.get("status", "UNKNOWN")
+                    }
+                    
+                    # Add to appropriate dataframe based on status
+                    if row["status"] == "FINISHED":
+                        finished_data.append(row)
+                    elif row["status"] == "RUNNING":
+                        running_data.append(row)
+                    elif row["status"] == "PENDING":
+                        pending_data.append(row)
+            except Exception as e:
+                print(f"Error reading request file {file_path}: {e}")
+                continue
+                
+        # Convert to dataframes
+        finished_df = pd.DataFrame(finished_data, columns=eval_cols)
+        running_df = pd.DataFrame(running_data, columns=eval_cols)
+        pending_df = pd.DataFrame(pending_data, columns=eval_cols)
+        
+        return finished_df, running_df, pending_df
+        
+    except Exception as e:
+        print(f"Error reading evaluation queue: {e}")
+        # Return empty dataframes
+        empty_df = pd.DataFrame(columns=eval_cols)
+        return empty_df.copy(), empty_df.copy(), empty_df.copy()

src/submission/check_validity.py

@@ -1,99 +1,113 @@
 import json
 import os
 import re
+import requests
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+from typing import Dict, Tuple, Any, List, Set
+
+def is_repository_valid(repo_name: str, repo_url: str) -> Tuple[bool, str, Dict[str, Any]]:
+    """
+    Checks if a GitHub repository is valid and accessible.
+    
+    Args:
+        repo_name: The name of the repository (org/repo format)
+        repo_url: URL to the repository
+        
+    Returns:
+        Tuple of (is_valid, error_message, library_info)
+    """
+    # Basic format validation
+    if not repo_name or "/" not in repo_name:
+        return False, "Repository name must be in the format 'organization/repository'", {}
+    
+    # Check if GitHub URL
+    if repo_url and "github.com" in repo_url:
+        # Extract org and repo from URL if provided
+        try:
+            parts = repo_url.split("github.com/")[1].split("/")
+            org = parts[0]
+            repo = parts[1].split(".")[0] if "." in parts[1] else parts[1]
+            url_repo_name = f"{org}/{repo}"
+            
+            # Check if URL matches repo_name
+            if url_repo_name != repo_name:
+                return False, f"Repository name ({repo_name}) doesn't match the URL ({url_repo_name})", {}
+        except:
+            pass  # Fall back to using repo_name
+    
+    # Get repository information from GitHub API
+    org, repo = repo_name.split("/")
+    api_url = f"https://api.github.com/repos/{org}/{repo}"
+    
     try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
+        response = requests.get(api_url)
+        if response.status_code != 200:
+            return False, f"Repository not found or not accessible: {response.json().get('message', 'Unknown error')}", {}
+        
+        # Parse repository data
+        repo_data = response.json()
+        library_info = get_library_info(repo_data)
+        
+        return True, "", library_info
+        
     except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
+        return False, f"Error accessing repository: {str(e)}", {}
+
+def get_library_info(repo_data: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Extracts relevant information from GitHub repository data.
+    
+    Args:
+        repo_data: GitHub API response for a repository
+        
+    Returns:
+        Dictionary with library metadata
+    """
+    library_info = {
+        "name": repo_data.get("name", ""),
+        "full_name": repo_data.get("full_name", ""),
+        "description": repo_data.get("description", ""),
+        "stars": repo_data.get("stargazers_count", 0),
+        "forks": repo_data.get("forks_count", 0),
+        "license": repo_data.get("license", {}).get("name", "Unknown"),
+        "created_at": repo_data.get("created_at", ""),
+        "updated_at": repo_data.get("updated_at", ""),
+        "open_issues": repo_data.get("open_issues_count", 0),
+        "default_branch": repo_data.get("default_branch", "main"),
+        "is_archived": repo_data.get("archived", False),
+    }
+    
+    return library_info
+
+def already_submitted_libraries(requested_libraries_dir: str) -> Tuple[Set[str], Dict[str, List[str]]]:
+    """
+    Gathers a list of already submitted libraries to avoid duplicates.
+    
+    Args:
+        requested_libraries_dir: Directory with library assessment requests
+        
+    Returns:
+        Tuple of (set of library identifiers, dict mapping orgs to submission dates)
+    """
     depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
+    library_ids = []
+    orgs_to_submission_dates = defaultdict(list)
 
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+    for root, _, files in os.walk(requested_libraries_dir):
+        current_depth = root.count(os.sep) - requested_libraries_dir.count(os.sep)
         if current_depth == depth:
             for file in files:
                 if not file.endswith(".json"):
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+                    library_ids.append(f"{info['library']}_{info['version']}")
 
                     # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                    if info["library"].count("/") == 0 or "submitted_time" not in info:
                         continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
+                    organisation, _ = info["library"].split("/")
+                    orgs_to_submission_dates[organisation].append(info["submitted_time"])
 
-    return set(file_names), users_to_submission_dates
+    return set(library_ids), orgs_to_submission_dates

src/submission/submit.py

@@ -1,119 +1,94 @@
 import json
 import os
-from datetime import datetime, timezone
+import re
+import time
+import uuid
+from datetime import datetime
+from pathlib import Path
 
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
+import huggingface_hub
+import requests
+from huggingface_hub import HfApi
 
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
+from src.display.utils import LibraryType, Language, AssessmentStatus
+from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO, TOKEN
+from src.submission.check_validity import is_repository_valid, get_library_info
 
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
 
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
+def add_new_eval(
+    library_name,
+    library_version,
+    repository_url,
+    language,
+    framework,
+    library_type_str,
+) -> str:
+    """
+    Adds a new library to the assessment queue.
+    
+    Args:
+        library_name: Name of the library (org/repo format)
+        library_version: Version of the library
+        repository_url: URL to the repository
+        language: Programming language
+        framework: Related framework/ecosystem
+        library_type_str: Type of AI library
+        
+    Returns:
+        A message indicating the status of the submission
+    """
+    # Check if valid repository
+    is_valid, validity_message, library_info = is_repository_valid(library_name, repository_url)
+    
+    if not is_valid:
+        return f"⚠️ Invalid submission: {validity_message}"
+
+    # Parse library type
+    library_type = LibraryType.from_str(library_type_str)
+    if library_type == LibraryType.Unknown:
+        return "⚠️ Please select a valid library type."
+
+    # Create a unique identifier for the submission
+    uid = uuid.uuid4().hex[:6]
+    timestamp = datetime.now().isoformat()
+    request_filename = f"{library_name.replace('/', '_')}_eval_request_{timestamp}_{uid}.json"
+    
+    # Stars count and license info from library_info if available
+    stars = library_info.get("stars", 0)
+    license_name = library_info.get("license", "unknown")
+    
+    # Create the assessment request JSON
+    assessment_request = {
+        "library": library_name,
+        "version": library_version,
+        "repository_url": repository_url,
+        "language": language,
+        "framework": framework,
+        "library_type": library_type.value.name,
+        "license": license_name,
+        "stars": stars,
         "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
+        "submitted_time": timestamp,
+        "last_updated": timestamp,
+        "assessment_id": uid
     }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )
+    
+    # Save the request 
+    os.makedirs(EVAL_REQUESTS_PATH, exist_ok=True)
+    with open(os.path.join(EVAL_REQUESTS_PATH, request_filename), "w") as f:
+        json.dump(assessment_request, f, indent=2)
+    
+    try:
+        # Push the file to the HF repo
+        path = Path(os.path.join(EVAL_REQUESTS_PATH, request_filename))
+        API.upload_file(
+            path_or_fileobj=path,
+            path_in_repo=request_filename,
+            repo_id=QUEUE_REPO,
+            repo_type="dataset",
+        )
+        
+        return f"✅ Library '{library_name}' (version {library_version}) has been added to the assessment queue! Assessment ID: {uid}"
+    
+    except Exception as e:
+        return f"Error uploading assessment request: {str(e)}"