Fix 'AutoEvalColumn' has no attribute

debug.py

@@ -0,0 +1,39 @@
+import pandas as pd
+from src.populate import get_leaderboard_df
+from src.display.utils import COLS, BENCHMARK_COLS
+from src.about import Tasks
+from src.leaderboard.read_evals import get_raw_eval_results
+
+print("Tasks definitions:")
+for task in Tasks:
+    print(f"- {task.name}: benchmark={task.value.benchmark}, metric={task.value.metric}, col_name={task.value.col_name}")
+
+print("\nBenchmark columns:", BENCHMARK_COLS)
+
+try:
+    # Get raw results first
+    raw_results = get_raw_eval_results("eval-results", "eval-queue")
+    print("\nRaw results:")
+    for result in raw_results:
+        print("\nResult:")
+        print("- eval_name:", result.eval_name)
+        print("- results:", result.results)
+        data_dict = result.to_dict()
+        print("- data_dict:", data_dict)
+    
+    # Convert to DataFrame
+    all_data_json = [v.to_dict() for v in raw_results]
+    df = pd.DataFrame.from_records(all_data_json)
+    print("\nDataFrame columns:", df.columns.tolist())
+    print("\nDataFrame contents:")
+    print(df)
+except Exception as e:
+    print("\nError:", str(e))
+    import traceback
+    traceback.print_exc()
+    
+    # Print raw data for debugging
+    print("\nRaw data from results file:")
+    import json
+    with open("eval-results/results_1.json") as f:
+        print(json.dumps(json.load(f), indent=2))

hub/version.txt

@@ -0,0 +1 @@
+1

src/about.py

@@ -15,7 +15,7 @@ class Tasks(Enum):
     safetensors = Task("safetensors_check", "compliant", "Safetensors")
     
     # Security prompts evaluation
-    secure_coding = Task("secure_coding", "security_score", "Security Score")
+    secure_coding = Task("secure_coding", "security_score", "Security Score ⬆️")
 
 NUM_FEWSHOT = 0
 # ---------------------------------------------------

src/leaderboard/read_evals.py

@@ -70,14 +70,14 @@ class EvalResult:
         results = {}
         for task in Tasks:
             task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
+            if task.benchmark in data["results"]:
+                result = data["results"][task.benchmark]
+                metric_value = result.get(task.metric)
+                if metric_value is not None:
+                    if isinstance(metric_value, bool):
+                        results[task.col_name] = metric_value
+                    else:
+                        results[task.col_name] = metric_value * 100.0
 
         return self(
             eval_name=result_key,
@@ -95,7 +95,7 @@ class EvalResult:
         """Finds the relevant request file for the current model and updates info with it"""
         request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
 
-        try:
+        if request_file:
             with open(request_file, "r") as f:
                 request = json.load(f)
             self.model_type = ModelType.from_str(request.get("model_type", ""))
@@ -112,30 +112,28 @@ class EvalResult:
             self.likes = request.get("likes", 0)
             self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
-        except Exception:
+        else:
             print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            "eval_name": self.eval_name,  # not a column, just a save name
+            "Precision": self.precision.value.name,
+            "Type": self.model_type.value.name,
+            "T": self.model_type.value.symbol,
+            "Weight Format": self.weight_type.value.name,
+            "Architecture": self.architecture,
+            "Model": make_clickable_model(self.full_model),
+            "Model SHA": self.revision,
+            "Hub License": self.license,
+            "Hub ❤️": self.likes,
+            "#Params (B)": self.num_params,
+            "Available on Hub": self.still_on_hub,
         }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+        
+        # Add benchmark results
+        data_dict.update(self.results)
 
         return data_dict
 
@@ -159,6 +157,7 @@ def get_request_file_for_model(requests_path, model_name, precision):
                 and req_content["precision"] == precision.split(".")[-1]
             ):
                 request_file = tmp_request_file
+                break
     return request_file
 
 
@@ -189,16 +188,14 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         # Store results of same eval together
         eval_name = eval_result.eval_name
         if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+            eval_results[eval_name].results.update(eval_result.results)
         else:
             eval_results[eval_name] = eval_result
 
     results = []
     for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
+        # Only include results that have all required benchmark values
+        if all(task.value.col_name in v.results for task in Tasks):
             results.append(v)
-        except KeyError:  # not all eval values present
-            continue
 
     return results

src/populate.py

@@ -14,11 +14,17 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by="Security Score ⬆️", ascending=False)  # Using the display name directly
+    
+    # Ensure all required columns exist before filtering
+    for col in benchmark_cols:
+        if col not in df.columns:
+            df[col] = None
+            
+    # Filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    df = df.sort_values(by="Security Score ⬆️", ascending=False)
     df = df[cols].round(decimals=2)
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
     return df
 
 
@@ -33,10 +39,17 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             with open(file_path) as fp:
                 data = json.load(fp)
 
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            # Create a new dict with the required column names
+            formatted_data = {
+                "model": make_clickable_model(data["model"]),
+                "revision": data.get("revision", "main"),
+                "private": data.get("private", False),
+                "precision": data.get("precision", ""),
+                "weight_type": data.get("weight_type", ""),
+                "status": data.get("status", "")
+            }
 
-            all_evals.append(data)
+            all_evals.append(formatted_data)
         elif ".md" not in entry:
             # this is a folder
             sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
@@ -45,9 +58,17 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 with open(file_path) as fp:
                     data = json.load(fp)
 
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
+                # Create a new dict with the required column names
+                formatted_data = {
+                    "model": make_clickable_model(data["model"]),
+                    "revision": data.get("revision", "main"),
+                    "private": data.get("private", False),
+                    "precision": data.get("precision", ""),
+                    "weight_type": data.get("weight_type", ""),
+                    "status": data.get("status", "")
+                }
+
+                all_evals.append(formatted_data)
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
     running_list = [e for e in all_evals if e["status"] == "RUNNING"]

test-locally.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Clean up any previous runs
+rm -rf venv eval-queue/* eval-results/* __pycache__ src/__pycache__ src/*/__pycache__
+
+# Create virtual environment
+python3 -m venv venv
+
+# Ensure we're using the virtual environment's Python and pip
+PYTHON="./venv/bin/python3"
+PIP="./venv/bin/pip"
+
+# Install dependencies
+$PYTHON -m pip install --upgrade pip
+$PIP install -r requirements.txt
+
+# Create necessary directories
+mkdir -p eval-queue eval-results
+
+# Create sample data files with correct column names matching Tasks definitions
+cat > eval-queue/test_model_eval_request_float16.json << EOL
+{
+    "model": "test/model",
+    "precision": "float16",
+    "model_type": "pretrained 🟢",
+    "weight_type": "Safetensors",
+    "license": "MIT",
+    "likes": 100,
+    "params": 7,
+    "submitted_time": "2024-01-01",
+    "status": "FINISHED"
+}
+EOL
+
+cat > eval-results/results_1.json << EOL
+{
+    "config": {
+        "model_name": "test/model",
+        "model_dtype": "float16",
+        "model_sha": "main"
+    },
+    "results": {
+        "secure_coding": {
+            "security_score": 0.85
+        },
+        "safetensors_check": {
+            "compliant": true
+        }
+    }
+}
+EOL
+
+# Set environment variables
+export HF_HOME="."
+export HF_TOKEN="dummy-token"  # The app will work locally without a real token
+
+# Run the app
+echo "Starting the app..."
+$PYTHON app.py