Change all print instances to logs

src/core/queue_manager.py

@@ -19,17 +19,17 @@ class QueueItem:
     request_id: str
     model_id: str
     revision: str
-    
+
 class QueueManager:
     """Manages evaluation request queue with persistence."""
-    
+
     def __init__(self, queue_dir: str):
         self.queue_dir = queue_dir
         self.queue = PriorityQueue()
         self.active_evaluations: List[str] = []
         self.lock = Lock()
         self._load_persisted_queue()
-        
+
     def _load_persisted_queue(self) -> None:
         """Load persisted queue items from disk."""
         try:
@@ -48,14 +48,14 @@ class QueueManager:
                 logger.info(f"Loaded {self.queue.qsize()} items from persisted queue")
         except Exception as e:
             logger.error(f"Failed to load persisted queue: {str(e)}")
-            
+
     def _persist_queue(self) -> None:
         """Persist current queue state to disk."""
         try:
             # Create a list of all queue items
             items = []
             temp_queue = PriorityQueue()
-            
+
             while not self.queue.empty():
                 item = self.queue.get()
                 items.append({
@@ -66,20 +66,20 @@ class QueueManager:
                     'revision': item.revision
                 })
                 temp_queue.put(item)
-                
+
             # Restore queue
             self.queue = temp_queue
-            
+
             # Save to disk
             os.makedirs(self.queue_dir, exist_ok=True)
             queue_file = os.path.join(self.queue_dir, "queue_state.json")
             with open(queue_file, 'w') as f:
                 json.dump(items, f, indent=2)
-                
+
             logger.info(f"Persisted {len(items)} items to queue state")
         except Exception as e:
             logger.error(f"Failed to persist queue: {str(e)}")
-            
+
     def add_request(self, model_id: str, revision: str, priority: int = 1) -> str:
         """Add new evaluation request to queue."""
         with self.lock:
@@ -95,7 +95,7 @@ class QueueManager:
             self._persist_queue()
             logger.info(f"Added request {request_id} to queue")
             return request_id
-            
+
     def get_next_request(self) -> Optional[QueueItem]:
         """Get next request from queue."""
         with self.lock:
@@ -106,14 +106,14 @@ class QueueManager:
                 logger.info(f"Retrieved request {item.request_id} from queue")
                 return item
             return None
-            
+
     def mark_complete(self, request_id: str) -> None:
         """Mark evaluation request as complete."""
         with self.lock:
             if request_id in self.active_evaluations:
                 self.active_evaluations.remove(request_id)
                 logger.info(f"Marked request {request_id} as complete")
-                
+
     def get_queue_status(self) -> dict:
         """Get current queue status."""
         with self.lock:

src/envs.py

@@ -8,9 +8,10 @@ TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
 
 # Change these to your organization name
 OWNER = "stacklok"  # Create this organization on HuggingFace
+BOARD_NAME = "secure_code_leaderboard"  # The name of the leaderboard board
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/secure-llm-leaderboard"
+REPO_ID = f"{OWNER}/{BOARD_NAME}"
 QUEUE_REPO = f"{OWNER}/requests"
 RESULTS_REPO = f"{OWNER}/results"
 

src/leaderboard/read_evals.py

@@ -43,14 +43,14 @@ class EvalResult:
 
         model_id = data.get("model_id", "")
         org_and_model = model_id.split("/", 1)
-        
+
         if len(org_and_model) == 1:
             org = None
             model = org_and_model[0]
         else:
             org = org_and_model[0]
             model = org_and_model[1]
-        
+
         full_model = model_id
         precision = Precision.from_str(data.get("precision", "Unknown"))
         result_key = f"{org}_{model}_{precision.value.name}" if org else f"{model}_{precision.value.name}"
@@ -78,7 +78,7 @@ class EvalResult:
             precision=precision,
             revision=data.get("revision", ""),
             still_on_hub=True,  # Assuming it's on the hub, you might want to check this
-            architecture="Unknown",  # You might want to add this information to your JSON if needed
+            architecture="Unknown",  # TODO: Need to get this from the model
             security_score=data.get("security_score", 0.0),
             safetensors_compliant=data.get("safetensors_compliant", False)
         )
@@ -91,7 +91,7 @@ class EvalResult:
             with open(request_file, "r") as f:
                 request = json.load(f)
             self.model_type = ModelType.from_str(request.get("model_type", ""))
-            
+
             # Handle weight type conversion from old format
             weight_type = request.get("weight_type", "PyTorch")
             if weight_type == "Original":
@@ -99,7 +99,7 @@ class EvalResult:
             elif weight_type == "Adapter":
                 weight_type = "Other"
             self.weight_type = WeightType[weight_type]
-            
+
             self.license = request.get("license", "?")
             self.likes = request.get("likes", 0)
             self.num_params = request.get("params", 0)
@@ -125,7 +125,7 @@ class EvalResult:
             "Security Score ⬆️": f"{self.security_score:.2f}",
             "Safetensors": str(self.safetensors_compliant)
         }
-        
+
         # Add benchmark results
         for key, value in self.results.items():
             data_dict[key] = str(value)
@@ -178,7 +178,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
                     logger.warning(f"Empty file {model_result_filepath}")
                     continue
                 data = json.loads(file_content)
-            
+
             if not data:
                 logger.warning(f"No data in file {model_result_filepath}")
                 continue
@@ -189,7 +189,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
             # Creation of result
             eval_result = EvalResult.init_from_json_file(model_result_filepath)
             logger.info(f"Created EvalResult object: {eval_result}")
-            
+
             eval_result.update_with_request_file(requests_path)
             logger.info(f"Updated EvalResult with request file: {eval_result}")
 
@@ -199,7 +199,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
                 eval_results[eval_name].results.update(eval_result.results)
             else:
                 eval_results[eval_name] = eval_result
-            
+
             logger.info(f"Processed evaluation result for {eval_name}")
         except json.JSONDecodeError as e:
             logger.error(f"Error decoding JSON in file {model_result_filepath}: {str(e)}")

src/leaderboard/security_eval.py

@@ -90,7 +90,7 @@ def get_model_response(
                 **inputs,
                 max_new_tokens=max_length,
                 num_return_sequences=1,
-                temperature=0.7 + (attempt * 0.1),  # Increase temperature slightly on retries
+                temperature=0.7,  # Increase temperature slightly on retries
                 do_sample=True,
                 pad_token_id=tokenizer.eos_token_id,
                 repetition_penalty=1.2,

src/populate.py

@@ -15,17 +15,17 @@ from src.config import RESULTS_REPO, QUEUE_REPO
 def get_leaderboard_df(cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     logger.info(f"Fetching evaluation results from {RESULTS_REPO}")
-    
+
     api = HfApi()
     all_data_json = []
 
     try:
         # List all files in the repository
         files = api.list_repo_files(repo_id=RESULTS_REPO, repo_type="dataset")
-        
+
         # Filter for JSON result files
         result_files = [f for f in files if f.endswith('_results.json')]
-        
+
         for file in result_files:
             try:
                 # Download and read each result file
@@ -73,13 +73,13 @@ def get_leaderboard_df(cols: list, benchmark_cols: list) -> pd.DataFrame:
 
     # Select only the columns we want to display
     df = df[cols]
-    
+
     # Round numeric columns
     numeric_cols = df.select_dtypes(include=[np.number]).columns
     for col in numeric_cols:
         df[col] = pd.to_numeric(df[col], errors='coerce')
     df[numeric_cols] = df[numeric_cols].round(decimals=2)
-    
+
     logger.debug(f"DataFrame after column selection and rounding:\n{df}")
 
     logger.info(f"Final DataFrame has {len(df)} rows")
@@ -96,10 +96,10 @@ def get_evaluation_queue_df(cols: list) -> list[pd.DataFrame]:
     try:
         # List all files in the repository
         files = api.list_repo_files(repo_id=QUEUE_REPO, repo_type="dataset")
-        
+
         # Filter for JSON files
         json_files = [f for f in files if f.endswith('.json')]
-        
+
         for file in json_files:
             try:
                 # Download and read each JSON file
@@ -123,12 +123,12 @@ def get_evaluation_queue_df(cols: list) -> list[pd.DataFrame]:
     except Exception as e:
         logger.error(f"Error fetching requests from {QUEUE_REPO}: {str(e)}", exc_info=True)
 
-    print(f"Found {len(all_evals)} total eval requests")
+    logger.info(f"Found {len(all_evals)} total eval requests")
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
     running_list = [e for e in all_evals if e["status"] == "RUNNING"]
     finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
 
-    print(f"Pending: {len(pending_list)}, Running: {len(running_list)}, Finished: {len(finished_list)}")
+    logger.info(f"Pending: {len(pending_list)}, Running: {len(running_list)}, Finished: {len(finished_list)}")
 
     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
     df_running = pd.DataFrame.from_records(running_list, columns=cols)

src/submission/check_validity.py

@@ -1,5 +1,6 @@
 import json
 import os
+import logging
 from collections import defaultdict
 
 import huggingface_hub
@@ -8,6 +9,8 @@ from huggingface_hub.hf_api import ModelInfo
 from transformers import AutoConfig
 from transformers.models.auto.tokenization_auto import AutoTokenizer
 
+logger = logging.getLogger(__name__)
+
 def check_model_card(repo_id: str) -> tuple[bool, str]:
     """Checks if the model card and license exist and have been filled"""
     try:
@@ -145,7 +148,7 @@ def already_submitted_models(requested_models_dir: str) -> tuple[set[str], defau
                         organisation, _ = model.split("/")
                         users_to_submission_dates[organisation].append(info["submitted_time"])
                 except (json.JSONDecodeError, KeyError, IOError) as e:
-                    print(f"Warning: Skipping malformed file {file}: {str(e)}")
+                    logger.warning(f"Skipping malformed file {file}: {str(e)}")
                     continue
 
     return set(file_names), users_to_submission_dates