app.py

import os
from pathlib import Path

import pandas as pd
import requests
import streamlit as st
from dotenv import load_dotenv

if Path(".env").is_file():
    load_dotenv(".env")


def download_submissions():
    auth_token = os.getenv("HF_HUB_TOKEN")
    header = {"Authorization": "Bearer " + auth_token}
    response = requests.get("http://huggingface.co/api/datasets", headers=header)
    all_datasets = response.json()

    submissions = []

    for dataset in all_datasets:
        is_benchmark = any([t for t in dataset["tags"] if t.split(":")[1] == "neelalex/raft-test"])
        if is_benchmark:
            submissions.append(dataset)

    for submission in submissions:
        submission_id = submission["id"]
        response = requests.get(
            f"http://huggingface.co/api/datasets/{submission_id}?full=true",
            headers=header,
        )
        data = response.json()
        submission_data = {"Team Name": [data["card_data"]["submission_dataset"].split("/")[0]]}
        for task in data["card_data"]["model-index"]:
            for task_name, splits in task.items():
                for split in splits:
                    if split["split"] == "test":
                        for metric in split["metrics"]:
                            if metric["name"] == "f1":
                                submission_data[task_name] = [metric["value"]]

    return pd.DataFrame(submission_data)


###########
### APP ###
###########
st.set_page_config(layout="wide")
st.title("RAFT Leaderboard")
df = download_submissions()
# hack to remove index column from https://github.com/streamlit/streamlit/issues/641
st.table(df.assign(hack="").set_index("hack"))