import json from pathlib import Path from typing import Dict, List, Optional import numpy as np import requests from fastapi import FastAPI, HTTPException, Query from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel class LeaderboardModel(BaseModel): model_name: str type: str model_link: Optional[str] = None scores: Dict[str, float] co2_cost: Optional[float] = None class LeaderboardData(BaseModel): models: List[LeaderboardModel] updated_at: str app = FastAPI( title="LLM Leaderboard API", description="API for serving Open LLM Leaderboard data", version="1.0.0" ) # Add CORS middleware to allow requests from your Gradio app app.add_middleware( CORSMiddleware, allow_origins=["*"], # For production, specify your exact frontend URL allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) # Cache for leaderboard data cached_data = None cache_file = Path("leaderboard_cache.json") def fetch_external_leaderboard_data(refresh: bool = False) -> Optional[Dict]: """ Fetch leaderboard data from external sources like HuggingFace. Uses local cache if available and refresh is False. """ global cached_data if not refresh and cached_data: return cached_data if not refresh and cache_file.exists(): try: with open(cache_file) as f: cached_data = json.load(f) return cached_data except: pass # Fall back to fetching if cache read fails try: # Try different endpoints that might contain leaderboard data endpoints = [ "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/raw/main/leaderboard_data.json", "https://huggingface.co/api/spaces/HuggingFaceH4/open_llm_leaderboard/api/get_results", ] for url in endpoints: response = requests.get(url) if response.status_code == 200: data = response.json() cached_data = data with open(cache_file, "w") as f: json.dump(data, f) return data # If all endpoints fail, return None return None except Exception as e: print(f"Error fetching external leaderboard data: {e}") return None def generate_sample_data() -> Dict: """ Generate sample leaderboard data when external data can't be fetched. """ models = [ {"model_name": "meta-llama/llama-3-70b-instruct", "type": "open"}, {"model_name": "mistralai/Mistral-7B-Instruct-v0.3", "type": "open"}, {"model_name": "google/gemma-7b-it", "type": "open"}, {"model_name": "Qwen/Qwen2-7B-Instruct", "type": "open"}, {"model_name": "anthropic/claude-3-opus", "type": "closed", "external_link": "https://www.anthropic.com/claude"}, {"model_name": "OpenAI/gpt-4o", "type": "closed", "external_link": "https://openai.com/gpt-4"}, {"model_name": "01-ai/Yi-1.5-34B-Chat", "type": "open"}, {"model_name": "google/gemma-2b", "type": "open"}, {"model_name": "microsoft/phi-3-mini-4k-instruct", "type": "open"}, {"model_name": "microsoft/phi-3-mini-128k-instruct", "type": "open"}, {"model_name": "stabilityai/stable-beluga-7b", "type": "open"}, {"model_name": "togethercomputer/RedPajama-INCITE-7B-Instruct", "type": "open"}, {"model_name": "databricks/dbrx-instruct", "type": "closed", "external_link": "https://www.databricks.com/product/machine-learning/large-language-models"}, {"model_name": "mosaicml/mpt-7b-instruct", "type": "open"}, {"model_name": "01-ai/Yi-1.5-9B-Chat", "type": "open"}, {"model_name": "anthropic/claude-3-sonnet", "type": "closed", "external_link": "https://www.anthropic.com/claude"}, {"model_name": "cohere/command-r-plus", "type": "closed", "external_link": "https://cohere.com/models/command-r-plus"}, {"model_name": "meta-llama/llama-3-8b-instruct", "type": "open"} ] np.random.seed(42) # For reproducibility model_data = [] for model_info in models: model_name = model_info["model_name"] model_type = model_info["type"] external_link = model_info.get("external_link", None) # Generate random scores average = round(np.random.uniform(40, 90), 2) ifeval = round(np.random.uniform(30, 90), 2) bbhi = round(np.random.uniform(40, 85), 2) math = round(np.random.uniform(20, 80), 2) gpqa = round(np.random.uniform(10, 70), 2) mujb = round(np.random.uniform(10, 70), 2) mmlu = round(np.random.uniform(40, 85), 2) co2_cost = round(np.random.uniform(1, 100), 2) # If it's an open model, it should have a link to Hugging Face model_link = None if external_link: model_link = external_link elif "/" in model_name: model_link = f"https://huggingface.co/{model_name}" else: model_link = f"https://huggingface.co/models?search={model_name}" model_data.append({ "model_name": model_name, "type": model_type, "model_link": model_link, "scores": { "average": average, "ifeval": ifeval, "bbhi": bbhi, "math": math, "gpqa": gpqa, "mujb": mujb, "mmlu": mmlu }, "co2_cost": co2_cost }) # Sort by average score model_data.sort(key=lambda x: x["scores"]["average"], reverse=True) # Create the final data structure from datetime import datetime leaderboard_data = { "models": model_data, "updated_at": datetime.now().isoformat() } return leaderboard_data @app.get("/") def read_root(): return {"message": "Welcome to the LLM Leaderboard API"} @app.get("/api/leaderboard", response_model=LeaderboardData) def get_leaderboard(refresh: bool = Query(False, description="Force refresh data from source")): """ Get the full leaderboard data. If refresh is True, force fetch from source instead of using cache. """ external_data = fetch_external_leaderboard_data(refresh=refresh) if external_data: # Process external data to match our expected format try: # Here you would transform the external data to match LeaderboardData model # This is a simplified example - you'd need to adapt this to the actual structure return external_data except Exception as e: print(f"Error processing external data: {e}") # Fall back to sample data if external data can't be processed return generate_sample_data() @app.get("/api/models", response_model=List[str]) def get_models(): """Get a list of all model names in the leaderboard""" data = fetch_external_leaderboard_data() or generate_sample_data() return [model["model_name"] for model in data["models"]] @app.get("/api/model/{model_name}", response_model=LeaderboardModel) def get_model_details(model_name: str): """Get detailed information about a specific model""" data = fetch_external_leaderboard_data() or generate_sample_data() for model in data["models"]: if model["model_name"] == model_name: return model raise HTTPException(status_code=404, detail=f"Model {model_name} not found") @app.get("/api/filters") def get_filter_counts(): """ Get counts for different filter categories to display in the UI. This matches what's shown in the 'Quick Filters' section of the leaderboard. """ data = fetch_external_leaderboard_data() or generate_sample_data() # Count models by different categories edge_count = 0 consumer_count = 0 midrange_count = 0 gpu_rich_count = 0 official_count = 0 for model in data["models"]: # Edge devices (typically small models) if "scores" in model and model["scores"].get("average", 0) < 45: edge_count += 1 # Consumer (moderate size/performance) if "scores" in model and 45 <= model["scores"].get("average", 0) < 55: consumer_count += 1 # Mid-range if "scores" in model and 55 <= model["scores"].get("average", 0) < 65: midrange_count += 1 # GPU-rich (high-end models) if "scores" in model and model["scores"].get("average", 0) >= 65: gpu_rich_count += 1 # Official providers # This is just a placeholder logic - adapt to your actual criteria if "/" not in model["model_name"] or model["model_name"].startswith("meta/") or model["model_name"].startswith("google/"): official_count += 1 return { "edge_devices": edge_count, "consumers": consumer_count, "midrange": midrange_count, "gpu_rich": gpu_rich_count, "official_providers": official_count } if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8000)