Spaces:
Running
Running
| import os, glob | |
| import json | |
| from datetime import datetime, timezone | |
| from dataclasses import dataclass | |
| from datasets import load_dataset, Dataset | |
| import pandas as pd | |
| import gradio as gr | |
| from huggingface_hub import HfApi, snapshot_download, ModelInfo, list_models | |
| from enum import Enum | |
| OWNER = "AIEnergyScore" | |
| COMPUTE_SPACE = f"{OWNER}/launch-computation-example" | |
| TOKEN = os.environ.get("DEBUG") | |
| API = HfApi(token=TOKEN) | |
| task_mappings = { | |
| 'automatic speech recognition': 'automatic-speech-recognition', | |
| 'Object Detection': 'object-detection', | |
| 'Text Classification': 'text-classification', | |
| 'Image to Text': 'image-to-text', | |
| 'Question Answering': 'question-answering', | |
| 'Text Generation': 'text-generation', | |
| 'Image Classification': 'image-classification', | |
| 'Sentence Similarity': 'sentence-similarity', | |
| 'Image Generation': 'image-generation', | |
| 'Summarization': 'summarization' | |
| } | |
| class ModelDetails: | |
| name: str | |
| display_name: str = "" | |
| symbol: str = "" # emoji | |
| def start_compute_space(): | |
| API.restart_space(COMPUTE_SPACE) | |
| gr.Info(f"Okay! {COMPUTE_SPACE} should be running now!") | |
| def get_model_size(model_info: ModelInfo): | |
| """Gets the model size from the configuration, or the model name if the configuration does not contain the information.""" | |
| try: | |
| model_size = round(model_info.safetensors["total"] / 1e9, 3) | |
| except (AttributeError, TypeError): | |
| return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py | |
| return model_size | |
| def add_docker_eval(zip_file): | |
| new_fid_list = zip_file.split("/") | |
| new_fid = new_fid_list[-1] | |
| if new_fid.endswith('.zip'): | |
| API.upload_file( | |
| path_or_fileobj=zip_file, | |
| repo_id="AIEnergyScore/tested_proprietary_models", | |
| path_in_repo='submitted_models/' + new_fid, | |
| repo_type="dataset", | |
| commit_message="Adding logs via submission Space.", | |
| token=TOKEN | |
| ) | |
| gr.Info('Uploaded logs to dataset! We will validate their validity and add them to the next version of the leaderboard.') | |
| else: | |
| gr.Info('You can only upload .zip files here!') | |
| def add_new_eval(repo_id: str, task: str): | |
| model_owner = repo_id.split("/")[0] | |
| model_name = repo_id.split("/")[1] | |
| current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | |
| requests = load_dataset("AIEnergyScore/requests_debug", split="test", token=TOKEN) | |
| requests_dset = requests.to_pandas() | |
| model_list = requests_dset[requests_dset['status'] == 'COMPLETED']['model'].tolist() | |
| task_models = list(API.list_models(filter=task_mappings[task])) | |
| task_model_names = [m.id for m in task_models] | |
| if repo_id in model_list: | |
| gr.Info('This model has already been run!') | |
| elif repo_id not in task_model_names: | |
| gr.Info("This model isn't compatible with the chosen task! Pick a different model-task combination") | |
| else: | |
| # Is the model info correctly filled? | |
| try: | |
| model_info = API.model_info(repo_id=repo_id) | |
| model_size = get_model_size(model_info=model_info) | |
| likes = model_info.likes | |
| except Exception: | |
| gr.Info("Could not find information for model %s" % (model_name)) | |
| model_size = None | |
| likes = None | |
| gr.Info("Adding request") | |
| request_dict = { | |
| "model": repo_id, | |
| "status": "PENDING", | |
| "submitted_time": pd.to_datetime(current_time), | |
| "task": task_mappings[task], | |
| "likes": likes, | |
| "params": model_size, | |
| "leaderboard_version": "v0", | |
| } | |
| print("Writing out request file to dataset") | |
| df_request_dict = pd.DataFrame([request_dict]) | |
| print(df_request_dict) | |
| df_final = pd.concat([requests_dset, df_request_dict], ignore_index=True) | |
| updated_dset = Dataset.from_pandas(df_final) | |
| updated_dset.push_to_hub("AIEnergyScore/requests_debug", split="test", token=TOKEN) | |
| gr.Info("Starting compute space at %s " % COMPUTE_SPACE) | |
| return start_compute_space() | |
| def print_existing_models(): | |
| requests = load_dataset("AIEnergyScore/requests_debug", split="test", token=TOKEN) | |
| requests_dset = requests.to_pandas() | |
| model_df = requests_dset[['model', 'status']] | |
| model_df = model_df[model_df['status'] == 'COMPLETED'] | |
| return model_df | |
| def highlight_cols(x): | |
| df = x.copy() | |
| df[df['status'] == 'COMPLETED'] = 'color: green' | |
| df[df['status'] == 'PENDING'] = 'color: orange' | |
| df[df['status'] == 'FAILED'] = 'color: red' | |
| return df | |
| # Applying the style function | |
| existing_models = print_existing_models() | |
| formatted_df = existing_models.style.apply(highlight_cols, axis=None) | |
| def get_leaderboard_models(): | |
| path = r'leaderboard_v0_data/energy' | |
| filenames = glob.glob(path + "/*.csv") | |
| data = [] | |
| for filename in filenames: | |
| data.append(pd.read_csv(filename)) | |
| leaderboard_data = pd.concat(data, ignore_index=True) | |
| return leaderboard_data[['model', 'task']] | |
| # A placeholder for get_zip_data_link() -- replace with your actual implementation if available. | |
| def get_zip_data_link(): | |
| return ( | |
| '<a href="https://example.com/download.zip" ' | |
| 'style="margin: 0 10px; text-decoration: none; font-weight: bold; font-size: 1.1em; ' | |
| 'color: black; font-family: \'Inter\', sans-serif;">Download Logs</a>' | |
| ) | |
| with gr.Blocks() as demo: | |
| # --- Header Links (at the very top, evenly spaced) --- | |
| gr.HTML(""" | |
| <style> | |
| .header-link { | |
| color: black !important; | |
| } | |
| @media (prefers-color-scheme: dark) { | |
| .header-link { | |
| color: white !important; | |
| } | |
| } | |
| </style> | |
| <div style="display: flex; justify-content: space-evenly; align-items: center; margin-bottom: 20px;"> | |
| <a class="header-link" href="https://huggingface.co/spaces/AIEnergyScore/leaderboard" style="text-decoration: none; font-weight: bold; font-size: 1.1em; font-family: 'Inter', sans-serif;">Leaderboard</a> | |
| <a class="header-link" href="https://huggingface.co/spaces/AIEnergyScore/Label" style="text-decoration: none; font-weight: bold; font-size: 1.1em; font-family: 'Inter', sans-serif;">Label Generator</a> | |
| <a class="header-link" href="https://huggingface.github.io/AIEnergyScore/#faq" style="text-decoration: none; font-weight: bold; font-size: 1.1em; font-family: 'Inter', sans-serif;">FAQ</a> | |
| <a class="header-link" href="https://huggingface.github.io/AIEnergyScore/#documentation" style="text-decoration: none; font-weight: bold; font-size: 1.1em; font-family: 'Inter', sans-serif;">Documentation</a> | |
| <a class="header-link" href="https://huggingface.co/spaces/AIEnergyScore/README/discussions" style="text-decoration: none; font-weight: bold; font-size: 1.1em; font-family: 'Inter', sans-serif;">Community</a> | |
| </div> | |
| """) | |
| # --- Logo (centered) --- | |
| gr.HTML(""" | |
| <div style="margin-top: 0px;"> | |
| <picture style="display: block; margin: 0 auto; max-width: 300px;"> | |
| <source media="(prefers-color-scheme: dark)" srcset="https://huggingface.co/spaces/AIEnergyScore/Leaderboard/resolve/main/logodark.png"> | |
| <img src="https://huggingface.co/spaces/AIEnergyScore/Leaderboard/resolve/main/logo.png" | |
| alt="Logo" | |
| style="display: block; margin: 0 auto; max-width: 300px; height: auto;"> | |
| </picture> | |
| </div> | |
| """) | |
| gr.Markdown('<div style="text-align: center;"><h2>Submission Portal</h2></div>') | |
| gr.Markdown('<div style="text-align: center;">If you want us to evaluate a model hosted on the 🤗 Hub, enter the model ID and choose the corresponding task from the dropdown list below, then click <strong>Run Analysis</strong> to launch the benchmarking process.</div>') | |
| gr.Markdown('<div style="text-align: center;">If you\'ve used the <a href="https://github.com/huggingface/AIEnergyScore/">Docker file</a> to run your own evaluation, please submit the resulting log files at the bottom of the page.</div>') | |
| gr.Markdown('<div style="text-align: center;">The <a href="https://huggingface.co/spaces/AIEnergyScore/Leaderboard">Project Leaderboard</a> will be updated on a biannual basis (last updated in February 2025).</div>') | |
| with gr.Row(): | |
| with gr.Column(): | |
| task = gr.Dropdown( | |
| choices=list(task_mappings.keys()), | |
| label="Choose a benchmark task", | |
| value='Text Generation', | |
| multiselect=False, | |
| interactive=True, | |
| ) | |
| with gr.Column(): | |
| model_name_textbox = gr.Textbox(label="Model name (user_name/model_name)") | |
| with gr.Row(): | |
| with gr.Column(): | |
| submit_button = gr.Button("Submit for Analysis") | |
| submission_result = gr.Markdown() | |
| submit_button.click( | |
| fn=add_new_eval, | |
| inputs=[model_name_textbox, task], | |
| outputs=submission_result, | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Accordion("Submit log files from a Docker run:", open=False): | |
| gr.Markdown(""" | |
| **⚠️ Warning: By uploading the zip file, you confirm that you have read and agree to the following terms:** | |
| - **Public Data Sharing:** You consent to the public sharing of the energy performance data derived from your submission. No additional information related to this model, including proprietary configurations, will be disclosed. | |
| - **Data Integrity:** You certify that the log files submitted are accurate, unaltered, and generated directly from testing your model as per the specified benchmarking procedures. | |
| - **Model Representation:** You affirm that the model tested and submitted is representative of the production-level version, including its level of quantization and any other relevant characteristics impacting energy efficiency and performance. | |
| """) | |
| file_output = gr.File(visible=False) | |
| u = gr.UploadButton("Upload a zip file with logs", file_count="single", interactive=True) | |
| u.upload(add_docker_eval, u, file_output) | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Accordion("Models that are in the latest leaderboard version:", open=False, visible=False): | |
| gr.Dataframe(get_leaderboard_models()) | |
| with gr.Accordion("Models that have been benchmarked recently:", open=False, visible=False): | |
| gr.Dataframe(formatted_df) | |
| demo.launch() |