Logging and other improvements

app.py

@@ -3,7 +3,13 @@ from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 import pandas as pd
+import os
+import logging
+from datetime import datetime
 
+from src.core.evaluation import EvaluationManager, EvaluationRequest
+from src.core.queue_manager import QueueManager
+from src.logging_config import setup_logging
 from src.about import (
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
@@ -24,29 +30,72 @@ from src.display.utils import (
     WeightType,
     Precision
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.envs import (
+    API, 
+    CACHE_PATH,
+    EVAL_REQUESTS_PATH, 
+    EVAL_RESULTS_PATH, 
+    QUEUE_REPO, 
+    REPO_ID, 
+    RESULTS_REPO, 
+    TOKEN
+)
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 
 
+# Setup logging
+setup_logging(log_dir="logs")
+logger = logging.getLogger('web')
+
+# Initialize managers
+evaluation_manager = EvaluationManager(
+    results_dir=EVAL_RESULTS_PATH,
+    backup_dir=os.path.join(CACHE_PATH, "eval-backups")
+)
+
+queue_manager = QueueManager(
+    queue_dir=os.path.join(CACHE_PATH, "eval-queue")
+)
+
 def restart_space():
+    """Restart the Hugging Face space."""
+    logger.info("Restarting space")
     API.restart_space(repo_id=REPO_ID)
 
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
+def initialize_space():
+    """Initialize the space by downloading required data."""
+    logger.info("Initializing space")
+    try:
+        logger.info(f"Downloading queue data from {QUEUE_REPO}")
+        snapshot_download(
+            repo_id=QUEUE_REPO,
+            local_dir=EVAL_REQUESTS_PATH,
+            repo_type="dataset",
+            tqdm_class=None,
+            etag_timeout=30,
+            token=TOKEN
+        )
+    except Exception as e:
+        logger.error(f"Failed to download queue data: {str(e)}")
+        restart_space()
+
+    try:
+        logger.info(f"Downloading results data from {RESULTS_REPO}")
+        snapshot_download(
+            repo_id=RESULTS_REPO,
+            local_dir=EVAL_RESULTS_PATH,
+            repo_type="dataset",
+            tqdm_class=None,
+            etag_timeout=30,
+            token=TOKEN
+        )
+    except Exception as e:
+        logger.error(f"Failed to download results data: {str(e)}")
+        restart_space()
+
+# Initialize space
+initialize_space()
 
 
 LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
@@ -57,12 +106,40 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
+def process_evaluation_queue():
+    """Process pending evaluation requests."""
+    logger.info("Processing evaluation queue")
+    while True:
+        request = queue_manager.get_next_request()
+        if not request:
+            break
+            
+        try:
+            # Convert queue request to evaluation request
+            eval_request = EvaluationRequest(
+                model_id=request.model_id,
+                revision=request.revision,
+                precision="float16",  # Default precision
+                weight_type="Safetensors",
+                submitted_time=request.timestamp
+            )
+            
+            # Run evaluation
+            results = evaluation_manager.run_evaluation(eval_request)
+            logger.info(f"Evaluation complete for {request.model_id}")
+            
+            # Mark request as complete
+            queue_manager.mark_complete(request.request_id)
+            
+        except Exception as e:
+            logger.error(f"Evaluation failed for {request.model_id}: {str(e)}")
+            # Keep request in active queue for retry
+
 def init_leaderboard(df):
     """Initialize the leaderboard with the given DataFrame."""
     if df is None or df.empty:
-        # Create an empty DataFrame with the required columns
         df = pd.DataFrame(columns=COLS)
-        print("Creating empty leaderboard - no evaluations completed yet")
+        logger.info("Creating empty leaderboard - no evaluations completed yet")
     
     # Create the leaderboard
     return gr.Dataframe(
@@ -183,8 +260,27 @@ with demo:
 
             submit_button = gr.Button("Submit for Security Evaluation")
             submission_result = gr.Markdown()
+            def handle_submission(model, base_model, revision, precision, weight_type, model_type):
+                """Handle new model submission."""
+                try:
+                    logger.info(f"New submission received for {model}")
+                    
+                    # Add to queue
+                    request_id = queue_manager.add_request(
+                        model_id=model,
+                        revision=revision if revision else "main"
+                    )
+                    
+                    # Process queue
+                    process_evaluation_queue()
+                    
+                    return gr.Markdown("Submission successful! Your model has been added to the evaluation queue.")
+                except Exception as e:
+                    logger.error(f"Submission failed: {str(e)}")
+                    return gr.Markdown(f"Error: {str(e)}")
+
             submit_button.click(
-                add_new_eval,
+                handle_submission,
                 [
                     model_name_textbox,
                     base_model_name_textbox,
@@ -206,7 +302,11 @@ with demo:
                 show_copy_button=True,
             )
 
+# Setup schedulers
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(process_evaluation_queue, "interval", seconds=300)  # Process queue every 5 minutes
 scheduler.start()
+
+logger.info("Application startup complete")
 demo.queue(default_concurrency_limit=40).launch()

src/core/evaluation.py

@@ -0,0 +1,105 @@
+"""Core evaluation logic separated from web interface."""
+import logging
+from typing import Dict, Any, Optional
+from dataclasses import dataclass
+from datetime import datetime
+import json
+import os
+
+from src.leaderboard.security_eval import (
+    check_safetensors,
+    evaluate_secure_coding,
+    load_model_and_tokenizer,
+)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler('evaluation.log'),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+@dataclass
+class EvaluationRequest:
+    """Data class for evaluation requests."""
+    model_id: str
+    revision: str
+    precision: str
+    weight_type: str
+    submitted_time: datetime
+    base_model: Optional[str] = None
+
+class EvaluationManager:
+    """Manages the evaluation pipeline and state."""
+    
+    def __init__(self, results_dir: str, backup_dir: str):
+        self.results_dir = results_dir
+        self.backup_dir = backup_dir
+        self._ensure_directories()
+        self.logger = logging.getLogger(__name__)
+    
+    def _ensure_directories(self) -> None:
+        """Ensure required directories exist."""
+        os.makedirs(self.results_dir, exist_ok=True)
+        os.makedirs(self.backup_dir, exist_ok=True)
+        
+    def backup_results(self, eval_id: str) -> None:
+        """Create backup of evaluation results."""
+        try:
+            result_path = os.path.join(self.results_dir, f"{eval_id}.json")
+            backup_path = os.path.join(self.backup_dir, f"{eval_id}_{datetime.now().isoformat()}.json")
+            
+            if os.path.exists(result_path):
+                with open(result_path, 'r') as src, open(backup_path, 'w') as dst:
+                    json.dump(json.load(src), dst, indent=2)
+                self.logger.info(f"Created backup at {backup_path}")
+        except Exception as e:
+            self.logger.error(f"Backup failed for {eval_id}: {str(e)}")
+
+    def run_evaluation(self, request: EvaluationRequest) -> Dict[str, Any]:
+        """Run evaluation pipeline for a model."""
+        try:
+            self.logger.info(f"Starting evaluation for {request.model_id}")
+            
+            # Run security checks
+            safetensors_compliant = check_safetensors(request.model_id, request.revision)
+            self.logger.info(f"Safetensors check: {safetensors_compliant}")
+            
+            # Load model for evaluation
+            model, tokenizer = load_model_and_tokenizer(request.model_id, request.revision)
+            self.logger.info("Model loaded successfully")
+            
+            # Run security evaluation
+            security_score = evaluate_secure_coding(request.model_id, request.revision)
+            self.logger.info(f"Security evaluation complete. Score: {security_score}")
+            
+            # Compile results
+            results = {
+                "model_id": request.model_id,
+                "revision": request.revision,
+                "evaluation_time": datetime.now().isoformat(),
+                "safetensors_compliant": safetensors_compliant,
+                "security_score": security_score,
+                "precision": request.precision,
+                "weight_type": request.weight_type,
+            }
+            
+            # Save and backup results
+            eval_id = f"{request.model_id.replace('/', '_')}_{request.revision}"
+            result_path = os.path.join(self.results_dir, f"{eval_id}.json")
+            
+            with open(result_path, 'w') as f:
+                json.dump(results, f, indent=2)
+            
+            self.backup_results(eval_id)
+            self.logger.info(f"Evaluation complete for {request.model_id}")
+            
+            return results
+            
+        except Exception as e:
+            self.logger.error(f"Evaluation failed for {request.model_id}: {str(e)}")
+            raise

src/core/queue_manager.py

@@ -0,0 +1,124 @@
+"""Queue management system for model evaluations."""
+import logging
+from typing import List, Optional
+from dataclasses import dataclass
+from datetime import datetime
+import json
+import os
+from queue import PriorityQueue
+import threading
+from threading import Lock
+
+logger = logging.getLogger(__name__)
+
+@dataclass(order=True)
+class QueueItem:
+    """Priority queue item for evaluations."""
+    priority: int
+    timestamp: datetime
+    request_id: str
+    model_id: str
+    revision: str
+    
+class QueueManager:
+    """Manages evaluation request queue with persistence."""
+    
+    def __init__(self, queue_dir: str):
+        self.queue_dir = queue_dir
+        self.queue = PriorityQueue()
+        self.active_evaluations: List[str] = []
+        self.lock = Lock()
+        self._load_persisted_queue()
+        
+    def _load_persisted_queue(self) -> None:
+        """Load persisted queue items from disk."""
+        try:
+            queue_file = os.path.join(self.queue_dir, "queue_state.json")
+            if os.path.exists(queue_file):
+                with open(queue_file, 'r') as f:
+                    items = json.load(f)
+                    for item in items:
+                        self.queue.put(QueueItem(
+                            priority=item['priority'],
+                            timestamp=datetime.fromisoformat(item['timestamp']),
+                            request_id=item['request_id'],
+                            model_id=item['model_id'],
+                            revision=item['revision']
+                        ))
+                logger.info(f"Loaded {self.queue.qsize()} items from persisted queue")
+        except Exception as e:
+            logger.error(f"Failed to load persisted queue: {str(e)}")
+            
+    def _persist_queue(self) -> None:
+        """Persist current queue state to disk."""
+        try:
+            # Create a list of all queue items
+            items = []
+            temp_queue = PriorityQueue()
+            
+            while not self.queue.empty():
+                item = self.queue.get()
+                items.append({
+                    'priority': item.priority,
+                    'timestamp': item.timestamp.isoformat(),
+                    'request_id': item.request_id,
+                    'model_id': item.model_id,
+                    'revision': item.revision
+                })
+                temp_queue.put(item)
+                
+            # Restore queue
+            self.queue = temp_queue
+            
+            # Save to disk
+            os.makedirs(self.queue_dir, exist_ok=True)
+            queue_file = os.path.join(self.queue_dir, "queue_state.json")
+            with open(queue_file, 'w') as f:
+                json.dump(items, f, indent=2)
+                
+            logger.info(f"Persisted {len(items)} items to queue state")
+        except Exception as e:
+            logger.error(f"Failed to persist queue: {str(e)}")
+            
+    def add_request(self, model_id: str, revision: str, priority: int = 1) -> str:
+        """Add new evaluation request to queue."""
+        with self.lock:
+            request_id = f"{model_id.replace('/', '_')}_{revision}_{datetime.now().isoformat()}"
+            item = QueueItem(
+                priority=priority,
+                timestamp=datetime.now(),
+                request_id=request_id,
+                model_id=model_id,
+                revision=revision
+            )
+            self.queue.put(item)
+            self._persist_queue()
+            logger.info(f"Added request {request_id} to queue")
+            return request_id
+            
+    def get_next_request(self) -> Optional[QueueItem]:
+        """Get next request from queue."""
+        with self.lock:
+            if not self.queue.empty():
+                item = self.queue.get()
+                self.active_evaluations.append(item.request_id)
+                self._persist_queue()
+                logger.info(f"Retrieved request {item.request_id} from queue")
+                return item
+            return None
+            
+    def mark_complete(self, request_id: str) -> None:
+        """Mark evaluation request as complete."""
+        with self.lock:
+            if request_id in self.active_evaluations:
+                self.active_evaluations.remove(request_id)
+                logger.info(f"Marked request {request_id} as complete")
+                
+    def get_queue_status(self) -> dict:
+        """Get current queue status."""
+        with self.lock:
+            return {
+                'queued': self.queue.qsize(),
+                'active': len(self.active_evaluations),
+                'active_evaluations': self.active_evaluations
+            }

src/logging_config.py

@@ -0,0 +1,71 @@
+"""Logging configuration for the application."""
+import logging
+import logging.handlers
+import os
+from datetime import datetime
+
+def setup_logging(
+    log_dir: str = "logs",
+    log_level: int = logging.INFO
+) -> None:
+    """
+    Configure application-wide logging.
+    
+    Args:
+        log_dir: Directory to store log files
+        log_level: Logging level to use
+    """
+    # Create logs directory
+    os.makedirs(log_dir, exist_ok=True)
+    
+    # Create formatters
+    file_formatter = logging.Formatter(
+        '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
+    console_formatter = logging.Formatter(
+        '%(levelname)s: %(message)s'
+    )
+    
+    # Setup file handler with rotation
+    log_file = os.path.join(
+        log_dir,
+        f"leaderboard_{datetime.now().strftime('%Y%m%d')}.log"
+    )
+    file_handler = logging.handlers.RotatingFileHandler(
+        log_file,
+        maxBytes=10485760,  # 10MB
+        backupCount=5
+    )
+    file_handler.setFormatter(file_formatter)
+    
+    # Setup console handler
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(console_formatter)
+    
+    # Setup root logger
+    root_logger = logging.getLogger()
+    root_logger.setLevel(log_level)
+    root_logger.addHandler(file_handler)
+    root_logger.addHandler(console_handler)
+    
+    # Create separate loggers for different components
+    loggers = {
+        'evaluation': logging.getLogger('evaluation'),
+        'queue': logging.getLogger('queue'),
+        'web': logging.getLogger('web'),
+        'security': logging.getLogger('security')
+    }
+    
+    # Configure component loggers
+    for name, logger in loggers.items():
+        logger.setLevel(log_level)
+        
+        # Create component-specific file handler
+        component_log = os.path.join(log_dir, f"{name}.log")
+        handler = logging.handlers.RotatingFileHandler(
+            component_log,
+            maxBytes=5242880,  # 5MB
+            backupCount=3
+        )
+        handler.setFormatter(file_formatter)
+        logger.addHandler(handler)

tests/test_evaluation.py

@@ -0,0 +1,103 @@
+"""Tests for core evaluation functionality."""
+import pytest
+import os
+import json
+from datetime import datetime
+from src.core.evaluation import EvaluationManager, EvaluationRequest
+
+@pytest.fixture
+def evaluation_manager(tmp_path):
+    """Create evaluation manager with temporary directories."""
+    results_dir = tmp_path / "results"
+    backup_dir = tmp_path / "backups"
+    return EvaluationManager(str(results_dir), str(backup_dir))
+
+def test_evaluation_manager_init(evaluation_manager):
+    """Test evaluation manager initialization."""
+    assert os.path.exists(evaluation_manager.results_dir)
+    assert os.path.exists(evaluation_manager.backup_dir)
+
+def test_backup_results(evaluation_manager):
+    """Test backup creation."""
+    # Create test results
+    eval_id = "test_model_main"
+    result_path = os.path.join(evaluation_manager.results_dir, f"{eval_id}.json")
+    test_results = {"test": "data"}
+    
+    os.makedirs(os.path.dirname(result_path), exist_ok=True)
+    with open(result_path, 'w') as f:
+        json.dump(test_results, f)
+        
+    # Create backup
+    evaluation_manager.backup_results(eval_id)
+    
+    # Check backup exists
+    backup_files = os.listdir(evaluation_manager.backup_dir)
+    assert len(backup_files) == 1
+    assert backup_files[0].startswith(eval_id)
+
+def test_run_evaluation(evaluation_manager):
+    """Test full evaluation run."""
+    request = EvaluationRequest(
+        model_id="hf-internal-testing/tiny-random-gpt2",
+        revision="main",
+        precision="float16",
+        weight_type="Safetensors",
+        submitted_time=datetime.now()
+    )
+    
+    results = evaluation_manager.run_evaluation(request)
+    
+    assert results["model_id"] == request.model_id
+    assert results["revision"] == request.revision
+    assert "security_score" in results
+    assert "safetensors_compliant" in results
+
+def test_evaluation_error_handling(evaluation_manager):
+    """Test error handling during evaluation."""
+    request = EvaluationRequest(
+        model_id="invalid/model",
+        revision="main",
+        precision="float16",
+        weight_type="Safetensors",
+        submitted_time=datetime.now()
+    )
+    
+    with pytest.raises(Exception):
+        evaluation_manager.run_evaluation(request)
+
+def test_concurrent_evaluations(evaluation_manager, tmp_path):
+    """Test handling of concurrent evaluations."""
+    import threading
+    import time
+    
+    def run_eval(model_id):
+        request = EvaluationRequest(
+            model_id=model_id,
+            revision="main",
+            precision="float16",
+            weight_type="Safetensors",
+            submitted_time=datetime.now()
+        )
+        try:
+            evaluation_manager.run_evaluation(request)
+        except Exception:
+            pass
+    
+    # Start multiple evaluation threads
+    threads = []
+    for i in range(3):
+        thread = threading.Thread(
+            target=run_eval,
+            args=(f"model_{i}",)
+        )
+        threads.append(thread)
+        thread.start()
+    
+    # Wait for all threads to complete
+    for thread in threads:
+        thread.join()
+    
+    # Check results directory integrity
+    assert os.path.exists(evaluation_manager.results_dir)
+    assert os.path.exists(evaluation_manager.backup_dir)

tests/test_queue.py

@@ -0,0 +1,128 @@
+"""Tests for queue management system."""
+import pytest
+import os
+import json
+from datetime import datetime
+from src.core.queue_manager import QueueManager, QueueItem
+
+@pytest.fixture
+def queue_manager(tmp_path):
+    """Create queue manager with temporary directory."""
+    queue_dir = tmp_path / "queue"
+    return QueueManager(str(queue_dir))
+
+def test_queue_manager_init(queue_manager):
+    """Test queue manager initialization."""
+    assert os.path.exists(queue_manager.queue_dir)
+    assert queue_manager.queue.empty()
+    assert len(queue_manager.active_evaluations) == 0
+
+def test_add_request(queue_manager):
+    """Test adding requests to queue."""
+    request_id = queue_manager.add_request("org/model", "main")
+    
+    assert not queue_manager.queue.empty()
+    assert os.path.exists(os.path.join(queue_manager.queue_dir, "queue_state.json"))
+    
+    # Verify persisted state
+    with open(os.path.join(queue_manager.queue_dir, "queue_state.json")) as f:
+        state = json.load(f)
+        assert len(state) == 1
+        assert state[0]["model_id"] == "org/model"
+
+def test_get_next_request(queue_manager):
+    """Test retrieving requests from queue."""
+    added_id = queue_manager.add_request("org/model", "main")
+    item = queue_manager.get_next_request()
+    
+    assert item is not None
+    assert item.model_id == "org/model"
+    assert item.revision == "main"
+    assert item.request_id in queue_manager.active_evaluations
+
+def test_mark_complete(queue_manager):
+    """Test marking requests as complete."""
+    added_id = queue_manager.add_request("org/model", "main")
+    item = queue_manager.get_next_request()
+    queue_manager.mark_complete(item.request_id)
+    
+    assert item.request_id not in queue_manager.active_evaluations
+
+def test_queue_status(queue_manager):
+    """Test queue status reporting."""
+    queue_manager.add_request("org/model1", "main")
+    queue_manager.add_request("org/model2", "main")
+    item = queue_manager.get_next_request()
+    
+    status = queue_manager.get_queue_status()
+    assert status["queued"] == 1
+    assert status["active"] == 1
+    assert item.request_id in status["active_evaluations"]
+
+def test_priority_ordering(queue_manager):
+    """Test priority-based queue ordering."""
+    # Add requests with different priorities
+    queue_manager.add_request("org/model1", "main", priority=2)
+    queue_manager.add_request("org/model2", "main", priority=1)  # Higher priority
+    queue_manager.add_request("org/model3", "main", priority=3)
+    
+    # First request should be model2 (priority 1)
+    item = queue_manager.get_next_request()
+    assert item.model_id == "org/model2"
+    
+    # Second should be model1 (priority 2)
+    item = queue_manager.get_next_request()
+    assert item.model_id == "org/model1"
+    
+    # Third should be model3 (priority 3)
+    item = queue_manager.get_next_request()
+    assert item.model_id == "org/model3"
+
+def test_queue_persistence(tmp_path):
+    """Test queue state persistence across instances."""
+    queue_dir = str(tmp_path / "queue")
+    
+    # Create first instance and add requests
+    manager1 = QueueManager(queue_dir)
+    manager1.add_request("org/model1", "main")
+    manager1.add_request("org/model2", "main")
+    
+    # Create second instance and verify state loaded
+    manager2 = QueueManager(queue_dir)
+    assert manager2.queue.qsize() == 2
+    
+    # Verify requests can be retrieved in correct order
+    item1 = manager2.get_next_request()
+    assert item1.model_id == "org/model1"
+    
+    item2 = manager2.get_next_request()
+    assert item2.model_id == "org/model2"
+
+def test_concurrent_access(queue_manager):
+    """Test concurrent queue access."""
+    import threading
+    import time
+    
+    def add_and_get():
+        # Add a request
+        queue_manager.add_request("org/model", "main")
+        time.sleep(0.1)  # Simulate some work
+        # Try to get a request
+        item = queue_manager.get_next_request()
+        if item:
+            queue_manager.mark_complete(item.request_id)
+    
+    # Create multiple threads
+    threads = []
+    for _ in range(5):
+        thread = threading.Thread(target=add_and_get)
+        threads.append(thread)
+        thread.start()
+    
+    # Wait for all threads to complete
+    for thread in threads:
+        thread.join()
+    
+    # Verify queue state is consistent
+    status = queue_manager.get_queue_status()
+    assert len(status["active_evaluations"]) == 0  # All should be marked complete