import gradio as gr import pandas as pd import os import zipfile import base64 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" CITATION_BUTTON_TEXT = r"""@misc{aienergyscore-leaderboard, author = {Sasha Luccioni and Boris Gamazaychikov and Emma Strubell and Sara Hooker and Yacine Jernite and Carole-Jean Wu and Margaret Mitchell}, title = {AI Energy Score Leaderboard - February 2025}, year = {2025}, publisher = {Hugging Face}, howpublished = "\url{https://huggingface.co/spaces/AIEnergyScore/Leaderboard}", }""" # List of tasks (CSV filenames) tasks = [ 'asr.csv', 'object_detection.csv', 'text_classification.csv', 'image_captioning.csv', 'question_answering.csv', 'text_generation.csv', 'image_classification.csv', 'sentence_similarity.csv', 'image_generation.csv', 'summarization.csv' ] def format_stars(score): try: score_int = int(score) except Exception: score_int = 0 # Render stars in black with a slightly larger font. return f'<span style="color: black; font-size:1.5em;">{"★" * score_int}</span>' def make_link(mname): parts = str(mname).split('/') display_name = parts[1] if len(parts) > 1 else mname return f'<a href="https://huggingface.co/{mname}" target="_blank">{display_name}</a>' def extract_link_text(html_link): """Extracts the inner text from an HTML link.""" start = html_link.find('>') + 1 end = html_link.rfind('</a>') if start > 0 and end > start: return html_link[start:end] else: return html_link def generate_html_table_from_df(df): """ Given a dataframe with a numeric energy column (gpu_energy_numeric), generate an HTML table with three columns: - Model (the link, with a fixed width based on the longest model name) - GPU Energy (Wh) plus a horizontal bar whose width is proportional to the energy value relative to the maximum in the table. - Score (displayed as stars) """ # Compute a static width (in pixels) for the Model column based on the longest model name. if not df.empty: max_length = max(len(extract_link_text(link)) for link in df['Model']) else: max_length = 10 # Multiply by an estimated average character width (10 pixels) and add some extra padding. static_width = max_length * 10 + 16 max_energy = df['gpu_energy_numeric'].max() if not df.empty else 1 color_map = {"1": "black", "2": "black", "3": "black", "4": "black", "5": "black"} html = '<table style="width:100%; border-collapse: collapse; font-family: Inter, sans-serif;">' # Keep only one header (the one with hover text) html += '<thead><tr style="background-color: #f2f2f2;">' html += '<th style="text-align: left; padding: 8px;" title="Model name with link to Hugging Face">Model</th>' html += '<th style="text-align: left; padding: 8px;" title="GPU energy consumed in Watt-hours for 1,000 queries">GPU Energy (Wh)</th>' html += '<th style="text-align: left; padding: 8px;" title="5 is most efficient, 1 is least. Relative energy efficiency score relative to other models in task/class at the time of leaderboard launch">Score</th>' html += '</tr></thead>' html += '<tbody>' for _, row in df.iterrows(): energy_numeric = row['gpu_energy_numeric'] energy_str = f"{energy_numeric:.2f}" # Compute the relative width (as a percentage) bar_width = (energy_numeric / max_energy) * 100 score_val = row['energy_score'] bar_color = color_map.get(str(score_val), "gray") html += '<tr>' html += f'<td style="padding: 8px; width: {static_width}px;">{row["Model"]}</td>' html += ( f'<td style="padding: 8px;">{energy_str}<br>' f'<div style="background-color: {bar_color}; width: {bar_width:.1f}%; height: 10px;"></div></td>' ) html += f'<td style="padding: 8px;">{row["Score"]}</td>' html += '</tr>' html += '</tbody></table>' return html # --- Function to zip all CSV files --- def zip_csv_files(): data_dir = "data/energy" zip_filename = "data.zip" with zipfile.ZipFile(zip_filename, "w", zipfile.ZIP_DEFLATED) as zipf: for filename in os.listdir(data_dir): if filename.endswith(".csv"): filepath = os.path.join(data_dir, filename) zipf.write(filepath, arcname=filename) return zip_filename def get_zip_data_link(): """Creates a data URI download link for the ZIP file.""" zip_filename = zip_csv_files() with open(zip_filename, "rb") as f: data = f.read() b64 = base64.b64encode(data).decode() href = f'<a href="data:application/zip;base64,{b64}" download="data.zip" style="margin: 0 15px; text-decoration: none; font-weight: bold; font-size: 1.1em;">Download Data</a>' return href # --- Modified functions to include a sort_order parameter --- def get_model_names_html(task, sort_order="Low to High"): df = pd.read_csv('data/energy/' + task) if df.columns[0].startswith("Unnamed:"): df = df.iloc[:, 1:] df['energy_score'] = df['energy_score'].astype(int) # Convert kWh to Wh: df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 df['Model'] = df['model'].apply(make_link) df['Score'] = df['energy_score'].apply(format_stars) ascending = True if sort_order == "Low to High" else False df = df.sort_values(by='gpu_energy_numeric', ascending=ascending) return generate_html_table_from_df(df) def get_all_model_names_html(sort_order="Low to High"): all_df = pd.DataFrame() for task in tasks: df = pd.read_csv('data/energy/' + task) if df.columns[0].startswith("Unnamed:"): df = df.iloc[:, 1:] df['energy_score'] = df['energy_score'].astype(int) df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 df['Model'] = df['model'].apply(make_link) df['Score'] = df['energy_score'].apply(format_stars) all_df = pd.concat([all_df, df], ignore_index=True) all_df = all_df.drop_duplicates(subset=['model']) ascending = True if sort_order == "Low to High" else False all_df = all_df.sort_values(by='gpu_energy_numeric', ascending=ascending) return generate_html_table_from_df(all_df) def get_text_generation_model_names_html(model_class, sort_order="Low to High"): df = pd.read_csv('data/energy/text_generation.csv') if df.columns[0].startswith("Unnamed:"): df = df.iloc[:, 1:] if 'class' in df.columns: df = df[df['class'] == model_class] df['energy_score'] = df['energy_score'].astype(int) df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 df['Model'] = df['model'].apply(make_link) df['Score'] = df['energy_score'].apply(format_stars) ascending = True if sort_order == "Low to High" else False df = df.sort_values(by='gpu_energy_numeric', ascending=ascending) return generate_html_table_from_df(df) # --- Update functions for dropdown changes --- # For Text Generation, two dropdowns: model class and sort order. def update_text_generation(selected_display, sort_order): mapping = { "A (Single Consumer GPU) <20B parameters": "A", "B (Single Cloud GPU) 20-66B parameters": "B", "C (Multiple Cloud GPUs) >66B parameters": "C" } model_class = mapping.get(selected_display, "A") return get_text_generation_model_names_html(model_class, sort_order) # For the other tabs, each update function simply takes the sort_order. def update_image_generation(sort_order): return get_model_names_html('image_generation.csv', sort_order) def update_text_classification(sort_order): return get_model_names_html('text_classification.csv', sort_order) def update_image_classification(sort_order): return get_model_names_html('image_classification.csv', sort_order) def update_image_captioning(sort_order): return get_model_names_html('image_captioning.csv', sort_order) def update_summarization(sort_order): return get_model_names_html('summarization.csv', sort_order) def update_asr(sort_order): return get_model_names_html('asr.csv', sort_order) def update_object_detection(sort_order): return get_model_names_html('object_detection.csv', sort_order) def update_sentence_similarity(sort_order): return get_model_names_html('sentence_similarity.csv', sort_order) def update_extractive_qa(sort_order): return get_model_names_html('question_answering.csv', sort_order) def update_all_tasks(sort_order): return get_all_model_names_html(sort_order) # --- Build the Gradio Interface --- demo = gr.Blocks(css=""" .gr-dataframe table { table-layout: fixed; width: 100%; } .gr-dataframe th, .gr-dataframe td { max-width: 150px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } """) with demo: # Replace title with a centered logo and a centered subtitle. gr.HTML('<div style="text-align: center;"><img src="https://huggingface.co/spaces/bgamazay/Leaderboard_test/resolve/main/logo.png" alt="Logo"></div>') gr.Markdown('<p style="text-align: center;">Welcome to the leaderboard for the <a href="https://huggingface.co/AIEnergyScore">AI Energy Score Project!</a> — Select different tasks to see scored models.</p>') # Header links (using a row of components, including a Download Data link) with gr.Row(): submission_link = gr.HTML('<a href="https://huggingface.co/spaces/AIEnergyScore/submission_portal" style="margin: 0 10px; text-decoration: none; font-weight: bold; font-size: 1.1em;">Submission Portal</a>') label_link = gr.HTML('<a href="https://huggingface.co/spaces/AIEnergyScore/Label" style="margin: 0 10px; text-decoration: none; font-weight: bold; font-size: 1.1em;">Label Generator</a>') faq_link = gr.HTML('<a href="https://huggingface.github.io/AIEnergyScore/#faq" style="margin: 0 10px; text-decoration: none; font-weight: bold; font-size: 1.1em;">FAQ</a>') documentation_link = gr.HTML('<a href="https://huggingface.github.io/AIEnergyScore/#documentation" style="margin: 0 10px; text-decoration: none; font-weight: bold; font-size: 1.1em;">Documentation</a>') download_link = gr.HTML(get_zip_data_link()) community_link = gr.HTML('<a href="https://huggingface.co/spaces/AIEnergyScore/README/discussions" style="margin: 0 10px; text-decoration: none; font-weight: bold; font-size: 1.1em;">Community</a>') with gr.Tabs(): # --- Text Generation Tab --- with gr.TabItem("Text Generation 💬"): with gr.Row(): model_class_options = [ "A (Single Consumer GPU) <20B parameters", "B (Single Cloud GPU) 20-66B parameters", "C (Multiple Cloud GPUs) >66B parameters" ] model_class_dropdown = gr.Dropdown( choices=model_class_options, label="Select Model Class", value=model_class_options[0] ) sort_dropdown_tg = gr.Dropdown( choices=["Low to High", "High to Low"], label="Sort", value="Low to High" ) tg_table = gr.HTML(get_text_generation_model_names_html("A", "Low to High")) # When either dropdown changes, update the table. model_class_dropdown.change(fn=update_text_generation, inputs=[model_class_dropdown, sort_dropdown_tg], outputs=tg_table) sort_dropdown_tg.change(fn=update_text_generation, inputs=[model_class_dropdown, sort_dropdown_tg], outputs=tg_table) # --- Image Generation Tab --- with gr.TabItem("Image Generation 📷"): sort_dropdown_img = gr.Dropdown( choices=["Low to High", "High to Low"], label="Sort", value="Low to High" ) img_table = gr.HTML(get_model_names_html('image_generation.csv', "Low to High")) sort_dropdown_img.change(fn=update_image_generation, inputs=sort_dropdown_img, outputs=img_table) # --- Text Classification Tab --- with gr.TabItem("Text Classification 🎭"): sort_dropdown_tc = gr.Dropdown( choices=["Low to High", "High to Low"], label="Sort", value="Low to High" ) tc_table = gr.HTML(get_model_names_html('text_classification.csv', "Low to High")) sort_dropdown_tc.change(fn=update_text_classification, inputs=sort_dropdown_tc, outputs=tc_table) # --- Image Classification Tab --- with gr.TabItem("Image Classification 🖼️"): sort_dropdown_ic = gr.Dropdown( choices=["Low to High", "High to Low"], label="Sort", value="Low to High" ) ic_table = gr.HTML(get_model_names_html('image_classification.csv', "Low to High")) sort_dropdown_ic.change(fn=update_image_classification, inputs=sort_dropdown_ic, outputs=ic_table) # --- Image Captioning Tab --- with gr.TabItem("Image Captioning 📝"): sort_dropdown_icap = gr.Dropdown( choices=["Low to High", "High to Low"], label="Sort", value="Low to High" ) icap_table = gr.HTML(get_model_names_html('image_captioning.csv', "Low to High")) sort_dropdown_icap.change(fn=update_image_captioning, inputs=sort_dropdown_icap, outputs=icap_table) # --- Summarization Tab --- with gr.TabItem("Summarization 📃"): sort_dropdown_sum = gr.Dropdown( choices=["Low to High", "High to Low"], label="Sort", value="Low to High" ) sum_table = gr.HTML(get_model_names_html('summarization.csv', "Low to High")) sort_dropdown_sum.change(fn=update_summarization, inputs=sort_dropdown_sum, outputs=sum_table) # --- Automatic Speech Recognition Tab --- with gr.TabItem("Automatic Speech Recognition 💬"): sort_dropdown_asr = gr.Dropdown( choices=["Low to High", "High to Low"], label="Sort", value="Low to High" ) asr_table = gr.HTML(get_model_names_html('asr.csv', "Low to High")) sort_dropdown_asr.change(fn=update_asr, inputs=sort_dropdown_asr, outputs=asr_table) # --- Object Detection Tab --- with gr.TabItem("Object Detection 🚘"): sort_dropdown_od = gr.Dropdown( choices=["Low to High", "High to Low"], label="Sort", value="Low to High" ) od_table = gr.HTML(get_model_names_html('object_detection.csv', "Low to High")) sort_dropdown_od.change(fn=update_object_detection, inputs=sort_dropdown_od, outputs=od_table) # --- Sentence Similarity Tab --- with gr.TabItem("Sentence Similarity 📚"): sort_dropdown_ss = gr.Dropdown( choices=["Low to High", "High to Low"], label="Sort", value="Low to High" ) ss_table = gr.HTML(get_model_names_html('sentence_similarity.csv', "Low to High")) sort_dropdown_ss.change(fn=update_sentence_similarity, inputs=sort_dropdown_ss, outputs=ss_table) # --- Extractive QA Tab --- with gr.TabItem("Extractive QA ❔"): sort_dropdown_qa = gr.Dropdown( choices=["Low to High", "High to Low"], label="Sort", value="Low to High" ) qa_table = gr.HTML(get_model_names_html('question_answering.csv', "Low to High")) sort_dropdown_qa.change(fn=update_extractive_qa, inputs=sort_dropdown_qa, outputs=qa_table) # --- All Tasks Tab --- with gr.TabItem("All Tasks 💡"): sort_dropdown_all = gr.Dropdown( choices=["Low to High", "High to Low"], label="Sort", value="Low to High" ) all_table = gr.HTML(get_all_model_names_html("Low to High")) sort_dropdown_all.change(fn=update_all_tasks, inputs=sort_dropdown_all, outputs=all_table) with gr.Accordion("📙 Citation", open=False): citation_button = gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, elem_id="citation-button", lines=10, show_copy_button=True, ) gr.Markdown("""Last updated: February 2025""") demo.launch()