import torch import random import torch.nn as nn from fastapi import APIRouter from datetime import datetime from datasets import load_dataset from sklearn.metrics import accuracy_score from transformers import AutoTokenizer, AutoModel, AutoConfig from huggingface_hub import hf_hub_download from safetensors.torch import load_file from .utils.evaluation import TextEvaluationRequest from .utils.emissions import tracker, clean_emissions_data, get_space_info DESCRIPTION = "GTE Architecture" ROUTE = "/text" class AutoBertClassifier(nn.Module): def __init__(self, num_labels=8, model_path="haisongzhang/roberta-tiny-cased"): super().__init__() self.tokenizer = AutoTokenizer.from_pretrained(model_path) self.bert = AutoModel.from_pretrained(model_path) self.config = AutoConfig.from_pretrained(model_path) self.config.num_labels = num_labels self.dropout = nn.Dropout(0.05) self.classifier = nn.Linear(self.bert.config.hidden_size, num_labels) def forward(self, input_ids, attention_mask): outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask) pooled_output = outputs.last_hidden_state[:, 0] # Using [CLS] token representation pooled_output = self.dropout(pooled_output) logits = self.classifier(pooled_output) return logits device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") model_repo = "elucidator8918/frugal-ai-text-tiny-final" model = AutoBertClassifier(num_labels=8) model.load_state_dict(load_file(hf_hub_download(repo_id=model_repo, filename="model.safetensors"))) tokenizer = AutoTokenizer.from_pretrained(model_repo) model = model.to(device) model.eval() router = APIRouter() @router.post(ROUTE, tags=["Text Task"], description=DESCRIPTION) async def evaluate_text(request: TextEvaluationRequest): """ Evaluate text classification for climate disinformation detection. Current Model: GTE Architecture """ # Get space info username, space_url = get_space_info() # Define the label mapping LABEL_MAPPING = { "0_not_relevant": 0, "1_not_happening": 1, "2_not_human": 2, "3_not_bad": 3, "4_solutions_harmful_unnecessary": 4, "5_science_unreliable": 5, "6_proponents_biased": 6, "7_fossil_fuels_needed": 7 } # Load and prepare the dataset dataset = load_dataset(request.dataset_name) # Convert string labels to integers dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]}) # Split dataset train_test = dataset["train"].train_test_split(test_size=request.test_size, seed=request.test_seed) test_dataset = train_test["test"] true_labels = test_dataset["label"] texts = test_dataset["quote"] # Start tracking emissions tracker.start() tracker.start_task("inference") #-------------------------------------------------------------------------------------------- # YOUR MODEL INFERENCE CODE HERE # Update the code below to replace the random baseline by your model inference within the inference pass where the energy consumption and emissions are tracked. #-------------------------------------------------------------------------------------------- text_encoding = tokenizer( texts, truncation=True, padding=True, return_tensors="pt", max_length=256 ) with torch.no_grad(): text_input_ids = text_encoding["input_ids"].to(device) text_attention_mask = text_encoding["attention_mask"].to(device) logits = model(text_input_ids, text_attention_mask) predictions = torch.argmax(logits, dim=1).cpu().numpy() #-------------------------------------------------------------------------------------------- # YOUR MODEL INFERENCE STOPS HERE #-------------------------------------------------------------------------------------------- # Stop tracking emissions emissions_data = tracker.stop_task() # Calculate accuracy accuracy = accuracy_score(true_labels, predictions) print(f"Accuracy = {accuracy}") # Prepare results dictionary results = { "username": username, "space_url": space_url, "submission_timestamp": datetime.now().isoformat(), "model_description": DESCRIPTION, "accuracy": float(accuracy), "energy_consumed_wh": emissions_data.energy_consumed * 1000, "emissions_gco2eq": emissions_data.emissions * 1000, "emissions_data": clean_emissions_data(emissions_data), "api_route": ROUTE, "dataset_config": { "dataset_name": request.dataset_name, "test_size": request.test_size, "test_seed": request.test_seed } } print(results) return results