uploads.py

import os
import json
import datetime
from email.utils import parseaddr
from io import BytesIO

from huggingface_hub import HfApi
import gradio as gr

from eval_utils import get_evaluation_scores

# Constants
LEADERBOARD_PATH = "Exploration-Lab/IL-TUR-Leaderboard"
SUBMISSION_FORMAT = "predictions"
TOKEN = os.environ.get("TOKEN", None)
YEAR_VERSION = "2024"

api = HfApi(token=TOKEN)


# Helper functions for formatting messages
def format_message(msg, color):
    return f"<p style='color: {color}; font-size: 20px; text-align: center;'>{msg}</p>"


def format_error(msg):
    return format_message(msg, "red")


def format_warning(msg):
    return format_message(msg, "orange")


def format_log(msg):
    return format_message(msg, "green")


def model_hyperlink(link, model_name):
    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'


def input_verification(method_name, url, path_to_file, organisation, mail):
    """Verify the input fields for submission."""
    # Check if any field is empty
    if any(
        input == "" for input in [method_name, url, path_to_file, organisation, mail]
    ):
        return format_warning("Please fill all the fields.")

    # Verify email format
    _, parsed_mail = parseaddr(mail)
    if "@" not in parsed_mail:
        return format_warning("Please provide a valid email address.")

    # Check if file is attached
    if path_to_file is None:
        return format_warning("Please attach a file.")

    return parsed_mail


def add_new_eval(
    method_name: str,
    submitted_by: str,
    url: str,
    path_to_file: str,
    organisation: str,
    mail: str,
):
    """Add a new evaluation to the leaderboard."""

    # Verify input
    parsed_mail = input_verification(method_name, url, path_to_file, organisation, mail)
    if parsed_mail.startswith("<p"):  # If it's a warning message
        return parsed_mail

    # Process submission
    if SUBMISSION_FORMAT == "predictions":
        # Read submission and gold data
        with open(path_to_file, "r") as f:
            submission_data = json.load(f)
        with open("submissions/baseline/IL_TUR_eval_gold_small.json", "r") as f:
            gold_data = json.load(f)

        # Get evaluation scores
        submission = get_evaluation_scores(gold_data, submission_data)
    else:
        # Read submission directly if it's not in predictions format
        with open(path_to_file, "r") as f:
            submission = json.load(f)

    # Update results
    with open("submissions/baseline/results.json", "r") as f:
        results = json.load(f)
    results.append(submission[0])

    # Prepare buffer for upload
    leaderboard_buffer = BytesIO(json.dumps(results).encode())
    leaderboard_buffer.seek(0)

    # Upload to Hugging Face
    api.upload_file(
        repo_id=LEADERBOARD_PATH,
        path_in_repo="submissions/baseline/results.json",
        path_or_fileobj=leaderboard_buffer,
        token=TOKEN,
        repo_type="space",
    )

    return format_log(
        f"Method {method_name} submitted by {organisation} successfully. \n"
        "Please refresh the leaderboard, and wait a bit to see the score displayed"
    )