functions.py

import gradio as gr
import pandas as pd
from datasets import load_dataset
from huggingface_hub import (
    CommitOperationAdd,
    EvalResult,
    ModelCard,
    RepoUrl,
    create_commit,
)
from huggingface_hub.repocard_data import eval_results_to_model_index
from pytablewriter import MarkdownTableWriter

COMMIT_DESCRIPTION = """This is an automated PR created with https://huggingface.co/spaces/T145/open-llm-leaderboard-results-to-modelcard

The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card.

Please report any issues here: https://huggingface.co/spaces/T145/open-llm-leaderboard-results-to-modelcard/discussions"""


def search(df, value):
    result_df = df[df["fullname"] == value]
    return result_df.iloc[0].to_dict() if not result_df.empty else None


def get_details_url(repo):
    author, model = repo.split("/")
    return f"https://huggingface.co/datasets/open-llm-leaderboard/{author}__{model}-details"


def get_contents_url(repo):
    return f"https://huggingface.co/datasets/open-llm-leaderboard/contents/viewer/default/train?q={repo}"


def get_query_url(repo):
    return f"https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query={repo}"


def get_task_summary(results):
    return {
        "IFEval": {
            "dataset_type": "wis-k/instruction-following-eval",
            "dataset_name": "IFEval (0-Shot)",
            "metric_type": "inst_level_strict_acc and prompt_level_strict_acc",
            "metric_value": round(results["IFEval"], 2),
            "dataset_config": None,
            "dataset_split": "train",
            "dataset_revision": None,
            "dataset_args": {"num_few_shot": 0},
            "metric_name": "averaged accuracy",
        },
        "BBH": {
            "dataset_type": "SaylorTwift/bbh",
            "dataset_name": "BBH (3-Shot)",
            "metric_type": "acc_norm",
            "metric_value": round(results["BBH"], 2),
            "dataset_config": None,
            "dataset_split": "test",
            "dataset_revision": None,
            "dataset_args": {"num_few_shot": 3},
            "metric_name": "normalized accuracy",
        },
        "MATH Lvl 5": {
            "dataset_type": "lighteval/MATH-Hard",
            "dataset_name": "MATH Lvl 5 (4-Shot)",
            "metric_type": "exact_match",
            "metric_value": round(results["MATH Lvl 5"], 2),
            "dataset_config": None,
            "dataset_split": "test",
            "dataset_revision": None,
            "dataset_args": {"num_few_shot": 4},
            "metric_name": "exact match",
        },
        "GPQA": {
            "dataset_type": "Idavidrein/gpqa",
            "dataset_name": "GPQA (0-shot)",
            "metric_type": "acc_norm",
            "metric_value": round(results["GPQA"], 2),
            "dataset_config": None,
            "dataset_split": "train",
            "dataset_revision": None,
            "dataset_args": {"num_few_shot": 0},
            "metric_name": "acc_norm",
        },
        "MuSR": {
            "dataset_type": "TAUR-Lab/MuSR",
            "dataset_name": "MuSR (0-shot)",
            "metric_type": "acc_norm",
            "metric_value": round(results["MUSR"], 2),
            "dataset_config": None,
            "dataset_split": None,  # three test splits
            "dataset_args": {"num_few_shot": 0},
            "metric_name": "acc_norm",
        },
        "MMLU-PRO": {
            "dataset_type": "TIGER-Lab/MMLU-Pro",
            "dataset_name": "MMLU-PRO (5-shot)",
            "metric_type": "acc",
            "metric_value": round(results["MMLU-PRO"], 2),
            "dataset_config": "main",
            "dataset_split": "test",
            "dataset_args": {"num_few_shot": 5},
            "metric_name": "accuracy",
        },
    }


def get_eval_results(df, repo):
    results = search(df, repo)
    task_summary = get_task_summary(results)
    table = MarkdownTableWriter()
    table.headers = ["Metric", "% Value"]
    table.value_matrix = [["Avg.", round(results["Average ⬆️"], 2)]] + [
        [v["dataset_name"], v["metric_value"]] for v in task_summary.values()
    ]

    text = f"""# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard)
Detailed results can be found [here]({get_details_url(repo)})!
Summarized results can be found [here]({get_contents_url(repo)})!

{table.dumps()}
"""
    return text


def get_edited_yaml_readme(df, repo, token: str | None):
    card = ModelCard.load(repo, token=token)
    results = search(df, repo)

    common = {
        "task_type": "text-generation",
        "task_name": "Text Generation",
        "source_name": "Open LLM Leaderboard",
        "source_url": f"https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query={repo}",
    }

    tasks_results = get_task_summary(results)

    if not card.data[
        "eval_results"
    ]:  # No results reported yet, we initialize the metadata
        card.data["model-index"] = eval_results_to_model_index(
            repo.split("/")[1],
            [EvalResult(**task, **common) for task in tasks_results.values()],
        )
    else:  # We add the new evaluations
        for task in tasks_results.values():
            cur_result = EvalResult(**task, **common)
            if any(
                result.is_equal_except_value(cur_result)
                for result in card.data["eval_results"]
            ):
                continue
            card.data["eval_results"].append(cur_result)

    return str(card)


def commit(
    repo,
    pr_number=None, # specify pr number if you want to edit it
    message="Adding Evaluation Results",
    oauth_token: gr.OAuthToken | None = None,
):
    if not oauth_token:
        raise gr.Warning("You are not logged in. Click on 'Sign in with Huggingface' to log in.")
    else:
        token = oauth_token

    data = load_dataset("open-llm-leaderboard/contents", split="train")
    df = pd.DataFrame(data)

    if repo.startswith("https://huggingface.co/"):
        try:
            repo = RepoUrl(repo).repo_id
        except Exception as e:
            raise gr.Error(f"Not a valid repo id: {str(repo)}") from e

    edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True}

    try:
        try:  # check if there is a readme already
            readme_text = get_edited_yaml_readme(
                df, repo, token=token
            ) + get_eval_results(df, repo)
        except Exception as e:
            if "Repo card metadata block was not found." in str(e):  # There is no readme
                readme_text = get_edited_yaml_readme(df, repo, token=token)
            else:
                print(f"Something went wrong: {e}")

        ops = [
            CommitOperationAdd(
                path_in_repo="README.md", path_or_fileobj=readme_text.encode()
            )
        ]
        commit = create_commit(
            repo_id=repo,
            token=token,
            operations=ops,
            commit_message=message,
            commit_description=COMMIT_DESCRIPTION,
            repo_type="model",
            **edited,
        ).pr_url

        return commit

    except Exception as e:
        if "Discussions are disabled for this repo" in str(e):
            return "Discussions disabled"
        elif "Cannot access gated repo" in str(e):
            return "Gated repo"
        elif "Repository Not Found" in str(e):
            return "Repository Not Found"
        else:
            return e