import gradio as gr import pandas as pd import plotly.express as px CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" CITATION_BUTTON_TEXT = r"""@misc{aienergyscore-leaderboard, author = {Sasha Luccioni and Boris Gamazaychikov and Emma Strubell and Sara Hooker and Yacine Jernite and Carole-Jean Wu and Margaret Mitchell}, title = {AI Energy Score Leaderboard - February 2025}, year = {2025}, publisher = {Hugging Face}, howpublished = "\url{https://huggingface.co/spaces/AIEnergyScore/Leaderboard}", }""" # List of tasks (CSV filenames) tasks = [ 'asr.csv', 'object_detection.csv', 'text_classification.csv', 'image_captioning.csv', 'question_answering.csv', 'text_generation.csv', 'image_classification.csv', 'sentence_similarity.csv', 'image_generation.csv', 'summarization.csv' ] def format_stars(score): try: score_int = int(score) except Exception: score_int = 0 return f'{"★" * score_int}' def make_link(mname): parts = str(mname).split('/') display_name = parts[1] if len(parts) > 1 else mname return f'[{display_name}](https://huggingface.co/{mname})' def get_plots(task): df = pd.read_csv('data/energy/' + task) if df.columns[0].startswith("Unnamed:"): df = df.iloc[:, 1:] df['energy_score'] = df['energy_score'].astype(int) df['Display Model'] = df['model'].apply(lambda m: m.split('/')[-1]) color_map = {1: "red", 2: "orange", 3: "yellow", 4: "lightgreen", 5: "green"} fig = px.scatter( df, x="total_gpu_energy", # Ensure correct column for x-axis y="Display Model", # Keep model name for y-axis color="energy_score", # Ensure correct column for point color custom_data=['energy_score'], height=500, width=800, color_discrete_map=color_map ) fig.update_traces( hovertemplate="
".join([ "Model: %{y}", "GPU Energy (Wh): %{x}", "Energy Score: %{customdata[0]}" ]) ) fig.update_layout(xaxis_title="GPU Energy (Wh)", yaxis_title="Model") return fig def get_all_plots(): all_df = pd.DataFrame() for task in tasks: df = pd.read_csv('data/energy/' + task) if df.columns[0].startswith("Unnamed:"): df = df.iloc[:, 1:] df['energy_score'] = df['energy_score'].astype(int) df['Display Model'] = df['model'].apply(lambda m: m.split('/')[-1]) all_df = pd.concat([all_df, df], ignore_index=True) all_df = all_df.drop_duplicates(subset=['model']) color_map = {1: "red", 2: "orange", 3: "yellow", 4: "lightgreen", 5: "green"} fig = px.scatter( all_df, x="total_gpu_energy", # Ensure correct column for x-axis y="Display Model", color="energy_score", # Ensure correct column for point color custom_data=['energy_score'], height=500, width=800, color_discrete_map=color_map ) fig.update_traces( hovertemplate="
".join([ "Model: %{y}", "GPU Energy (Wh): %{x}", "Energy Score: %{customdata[0]}" ]) ) fig.update_layout(xaxis_title="GPU Energy (Wh)", yaxis_title="Model") return fig def get_model_names(task): """ For a given task, load the energy CSV and return a dataframe with the following columns: - Model (a markdown link) - GPU Energy (Wh) formatted as a string with 4 decimal places - Score (a star rating based on energy_score) For text_generation.csv only, also add the "Class" column from the CSV. The final column order is: Model, GPU Energy (Wh), Score, [Class]. """ df = pd.read_csv('data/energy/' + task) if df.columns[0].startswith("Unnamed:"): df = df.iloc[:, 1:] df['energy_score'] = df['energy_score'].astype(int) # Format the energy as a string with 4 decimals df['GPU Energy (Wh)'] = df['total_gpu_energy'].apply(lambda x: f"{x:.4f}") df['Model'] = df['model'].apply(make_link) df['Score'] = df['energy_score'].apply(format_stars) if 'class' in df.columns: df['Class'] = df['class'] df = df[['Model', 'GPU Energy (Wh)', 'Score', 'Class']] else: df = df[['Model', 'GPU Energy (Wh)', 'Score']] df = df.sort_values(by='GPU Energy (Wh)') return df def get_all_model_names(): """ Combine data from all tasks and return a leaderboard table with: - Model, GPU Energy (Wh), Score Duplicate models are dropped. """ all_df = pd.DataFrame() for task in tasks: df = pd.read_csv('data/energy/' + task) if df.columns[0].startswith("Unnamed:"): df = df.iloc[:, 1:] df['energy_score'] = df['energy_score'].astype(int) df['GPU Energy (Wh)'] = df['total_gpu_energy'].apply(lambda x: f"{x:.4f}") df['Model'] = df['model'].apply(make_link) df['Score'] = df['energy_score'].apply(format_stars) all_df = pd.concat([all_df, df], ignore_index=True) all_df = all_df.drop_duplicates(subset=['model']) all_df = all_df.sort_values(by='GPU Energy (Wh)') return all_df[['Model', 'GPU Energy (Wh)', 'Score']] # Build the Gradio interface. # The css argument below makes all tables (e.g. leaderboard) use a fixed layout with narrower columns. demo = gr.Blocks(css=""" .gr-dataframe table { table-layout: fixed; width: 100%; } .gr-dataframe th, .gr-dataframe td { max-width: 150px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } """) with demo: gr.Markdown( """# AI Energy Score Leaderboard ### Welcome to the leaderboard for the [AI Energy Score Project!](https://huggingface.co/AIEnergyScore) Click through the tasks below to see how different models measure up in terms of energy efficiency.""" ) gr.Markdown( """Test your own models via the [submission portal](https://huggingface.co/spaces/AIEnergyScore/submission_portal)""" ) with gr.Tabs(): with gr.TabItem("Text Generation 💬"): with gr.Row(): with gr.Column(scale=1.3): plot = gr.Plot(get_plots('text_generation.csv')) with gr.Column(scale=1): table = gr.Dataframe(get_model_names('text_generation.csv'), datatype="markdown") with gr.TabItem("Image Generation 📷"): with gr.Row(): with gr.Column(): plot = gr.Plot(get_plots('image_generation.csv')) with gr.Column(): table = gr.Dataframe(get_model_names('image_generation.csv'), datatype="markdown") with gr.TabItem("Text Classification 🎭"): with gr.Row(): with gr.Column(): plot = gr.Plot(get_plots('text_classification.csv')) with gr.Column(): table = gr.Dataframe(get_model_names('text_classification.csv'), datatype="markdown") with gr.TabItem("Image Classification 🖼️"): with gr.Row(): with gr.Column(): plot = gr.Plot(get_plots('image_classification.csv')) with gr.Column(): table = gr.Dataframe(get_model_names('image_classification.csv'), datatype="markdown") with gr.TabItem("Image Captioning 📝"): with gr.Row(): with gr.Column(): plot = gr.Plot(get_plots('image_captioning.csv')) with gr.Column(): table = gr.Dataframe(get_model_names('image_captioning.csv'), datatype="markdown") with gr.TabItem("Summarization 📃"): with gr.Row(): with gr.Column(): plot = gr.Plot(get_plots('summarization.csv')) with gr.Column(): table = gr.Dataframe(get_model_names('summarization.csv'), datatype="markdown") with gr.TabItem("Automatic Speech Recognition 💬"): with gr.Row(): with gr.Column(): plot = gr.Plot(get_plots('asr.csv')) with gr.Column(): table = gr.Dataframe(get_model_names('asr.csv'), datatype="markdown") with gr.TabItem("Object Detection 🚘"): with gr.Row(): with gr.Column(): plot = gr.Plot(get_plots('object_detection.csv')) with gr.Column(): table = gr.Dataframe(get_model_names('object_detection.csv'), datatype="markdown") with gr.TabItem("Sentence Similarity 📚"): with gr.Row(): with gr.Column(): plot = gr.Plot(get_plots('sentence_similarity.csv')) with gr.Column(): table = gr.Dataframe(get_model_names('sentence_similarity.csv'), datatype="markdown") with gr.TabItem("Extractive QA ❔"): with gr.Row(): with gr.Column(): plot = gr.Plot(get_plots('question_answering.csv')) with gr.Column(): table = gr.Dataframe(get_model_names('question_answering.csv'), datatype="markdown") with gr.TabItem("All Tasks 💡"): with gr.Row(): with gr.Column(): plot = gr.Plot(get_all_plots) with gr.Column(): table = gr.Dataframe(get_all_model_names, datatype="markdown") with gr.Accordion("📙 Citation", open=False): citation_button = gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, elem_id="citation-button", lines=10, show_copy_button=True, ) gr.Markdown( """Last updated: February 2025""" ) demo.launch()