import gradio as gr from codecarbon import EmissionsTracker from datasets import load_dataset import numpy as np from sklearn.metrics import accuracy_score import random import os import json from datetime import datetime from huggingface_hub import HfApi from huggingface_hub import upload_file import tempfile from dotenv import load_dotenv import spaces # Use dotenv to load the environment variables load_dotenv() # Get HF token from environment variable HF_TOKEN = os.getenv("HF_TOKEN_TEXT") print(HF_TOKEN) if not HF_TOKEN: print("Warning: HF_TOKEN not found in environment variables. Submissions will not work.") # Initialize carbon emissions tracker with CodeCarbon tracker = EmissionsTracker(allow_multiple_runs=True) #-------------------------------------------------------------------------------------------- # FUNCTION TO UPDATE WITH YOUR MODEL SUBMISSION #-------------------------------------------------------------------------------------------- @spaces.GPU def evaluate(model_description): # Get space info username, space_url = get_space_info() # Initialize tracker tracker.start() tracker.start_task("inference") #-------------------------------------------------------------------------------------------- # YOUR MODEL INFERENCE CODE HERE # Update the code below to replace the random baseline by your model inference within the inference pass where the energy consumption and emissions are tracked. #-------------------------------------------------------------------------------------------- # Make random predictions true_labels = test_dataset["label"] predictions = [random.randint(0, 7) for _ in range(len(true_labels))] #-------------------------------------------------------------------------------------------- # YOUR MODEL INFERENCE STOPS HERE #-------------------------------------------------------------------------------------------- # Stop tracking emissions emissions_data = tracker.stop_task() # Calculate accuracy accuracy = accuracy_score(true_labels, predictions) # Prepare complete results results = { "username": username, "space_url": space_url, "submission_timestamp": datetime.now().isoformat(), "model_description": model_description if model_description else "No description provided", "accuracy": float(accuracy), "energy_consumed_wh": emissions_data.energy_consumed * 1000, "emissions_gco2eq": emissions_data.emissions * 1000, "emissions_data": clean_emissions_data(emissions_data) } # Return both summary and detailed results return [ accuracy, emissions_data.emissions * 1000, emissions_data.energy_consumed * 1000, json.dumps(results, indent=2) ] #-------------------------------------------------------------------------------------------- # HELPER FUNCTIONS #-------------------------------------------------------------------------------------------- # Function to get space username and URL def get_space_info(): space_name = os.getenv("SPACE_ID", "") if space_name: try: username = space_name.split("/")[0] space_url = f"https://huggingface.co/spaces/{space_name}" return username, space_url except Exception as e: print(f"Error getting space info: {e}") return "local-user", "local-development" def clean_emissions_data(emissions_data): """Remove unwanted fields from emissions data""" data_dict = emissions_data.__dict__ fields_to_remove = ['timestamp', 'project_name', 'experiment_id', 'latitude', 'longitude'] return {k: v for k, v in data_dict.items() if k not in fields_to_remove} def submit_results(results_json): if not results_json: return gr.Warning("No results to submit") # Check if we're in a Space or have admin dev rights space_name = os.getenv("SPACE_ID") is_admin_dev = os.getenv("ADMIN_DEV") == "true" if not space_name and not is_admin_dev: message = "You cannot submit your model locally, you need to deploy it as a Hugging Face Space first, and then submit it." return gr.Warning(message) if not HF_TOKEN: return gr.Warning("HF_TOKEN not found. Please set up your Hugging Face token.") try: # results_json is already a dict from gr.JSON results_str = json.dumps(results_json) # Create a temporary file with the results with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as f: f.write(results_str) temp_path = f.name # Upload to the dataset api = HfApi(token=HF_TOKEN) path_in_repo = f"submissions/{results_json['username']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" api.upload_file( path_or_fileobj=temp_path, path_in_repo=path_in_repo, repo_id="frugal-ai-challenge/public-leaderboard-text", repo_type="dataset", token=HF_TOKEN ) # Clean up os.unlink(temp_path) return gr.Info("Results submitted successfully to the leaderboard! 🎉") except Exception as e: return gr.Warning(f"Error submitting results: {str(e)}") #-------------------------------------------------------------------------------------------- # DATASET PREPARATION #-------------------------------------------------------------------------------------------- # Define the label mapping LABEL_MAPPING = { "0_not_relevant": 0, # No relevant claim detected "1_not_happening": 1, # Global warming is not happening "2_not_human": 2, # Not caused by humans "3_not_bad": 3, # Not bad or beneficial "4_solutions_harmful_unnecessary": 4, # Solutions harmful/unnecessary "5_science_unreliable": 5, # Science is unreliable "6_proponents_biased": 6, # Proponents are biased "7_fossil_fuels_needed": 7 # Fossil fuels are needed } # Load and prepare the dataset print("Loading dataset...") dataset = load_dataset("QuotaClimat/frugalaichallenge-text-train") # Convert string labels to integers dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]}) # Split dataset train_test = dataset["train"].train_test_split(test_size=0.2, seed=42) train_dataset = train_test["train"] test_dataset = train_test["test"] #-------------------------------------------------------------------------------------------- # GRADIO INTERFACE #-------------------------------------------------------------------------------------------- # Create the demo interface with gr.Blocks() as demo: gr.Image("./logo.png", show_label=False, container=False) gr.Markdown(""" # 📜 Frugal AI Challenge - Text task - Submission portal ## Climate Disinformation Classification """) with gr.Tabs(): with gr.Tab("Instructions"): gr.Markdown(""" To submit your results, please follow the steps below: ## Prepare your model submission 1. Clone the space of this portal on your own Hugging Face account. 2. Modify the ``evaluate`` function to replace the baseline by your model loading and inference within the inference pass where the energy consumption and emissions are tracked. 3. Eventually complete the requirements and/or any necessaries dependencies in your space. 4. Write down your model card in the ``modelcard.md`` file. 5. Deploy your space and verify that it works. 6. (Optional) You can change the Space hardware to use any GPU directly on Hugging Face. ## Submit your model to the leaderboard in the ``Model Submission`` tab 7. Step 1 - Evaluate model: Click on the button to evaluate your model. This will run you model, computes the accuracy on the test set (20% of the train set), and track the energy consumption and emissions. 8. Step 2 - Submit to leaderboard: Click on the button to submit your results to the leaderboard. This will upload the results to the leaderboard dataset and update the leaderboard. 9. You can see the leaderboard at https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-text ## About > You can find more information about the Frugal AI Challenge 2025 on the [Frugal AI Challenge website](https://frugal-ai-challenge.org/). > Or directly on the organization page on Hugging Face: [Frugal AI Challenge](https://huggingface.co/frugal-ai-challenge) This portal is a submission portal for the Frugal AI Challenge 2025. It is a simple interface to evaluate and submit your model to the leaderboard. The challenge is organized by Hugging Face, Data For Good, and the French Ministry of Environment. The goal of the Frugal AI Challenge is to encourage both academic and industry actors to keep efficiency in mind when deploying AI models. By tracking both energy consumption and performance for different AI tasks, we can incentivize frugality in AI deployment while also addressing real-world challenges. """) with gr.Tab("Model Submission"): with gr.Row(): model_description = gr.Textbox( label="Model Description (one sentence)", placeholder="Describe your model in one sentence...", value="Random baseline", lines=2 ) with gr.Row(): with gr.Column(scale=1): evaluate_btn = gr.Button("1. Evaluate model", variant="secondary") with gr.Column(scale=1): submit_btn = gr.Button("2. Submit to leaderboard", variant="primary", size="lg") with gr.Row(): accuracy_output = gr.Number(label="Accuracy", precision=4) energy_output = gr.Number(label="Energy Consumed (Wh)", precision=12) emissions_output = gr.Number(label="Emissions (gCO2eq)", precision=12) with gr.Row(): results_json = gr.JSON(label="Detailed Results", visible=True) evaluate_btn.click( evaluate, inputs=[model_description], outputs=[accuracy_output, emissions_output, energy_output, results_json] ) submit_btn.click( submit_results, inputs=[results_json], outputs=None # No need for output component with popups ) with gr.Tab("Model Card"): with open("README.md", "r") as f: content = f.read() # Remove the YAML header (content between --- markers) if content.startswith("---"): second_marker = content.find("---", 3) if second_marker != -1: content = content[second_marker + 3:].strip() gr.Markdown(content) if __name__ == "__main__": demo.launch()