Spaces:
Runtime error
Runtime error
File size: 4,276 Bytes
2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 |
import requests
import pandas as pd
from tqdm.auto import tqdm
import streamlit as st
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
def make_clickable(model_name):
link = "https://huggingface.co/" + model_name
return f'<a target="_blank" href="{link}">{model_name}</a>'
def get_model_ids():
api = HfApi()
# TODO: switch to hf-leaderboards for the final version.
models = api.list_models(filter="model-index")
model_ids = [x.modelId for x in models]
return model_ids
def get_metadata(model_id):
try:
readme_path = hf_hub_download(model_id, filename="README.md")
return metadata_load(readme_path)
except Exception:
# 404 README.md not found or problem loading it
return None
def parse_metric_value(value):
if isinstance(value, str):
"".join(value.split("%"))
try:
value = float(value)
except: # noqa: E722
value = None
elif isinstance(value, list):
if len(value) > 0:
value = value[0]
else:
value = None
value = round(value, 2) if isinstance(value, float) else None
return value
def parse_metrics_rows(meta):
if not isinstance(meta["model-index"], list) or len(meta["model-index"]) == 0 or "results" not in meta["model-index"][0]:
return None
for result in meta["model-index"][0]["results"]:
if "dataset" not in result or "metrics" not in result or "type" not in result["dataset"]:
continue
dataset = result["dataset"]["type"]
if "args" not in result["dataset"]:
continue
row = {"dataset": dataset}
for metric in result["metrics"]:
type = metric["type"].lower().strip()
value = parse_metric_value(metric.get("value", None))
if value is None:
continue
if type not in row or value < row[type]:
# overwrite the metric if the new value is lower (e.g. with LM)
row[type] = value
yield row
@st.cache(ttl=86400)
def get_data():
data = []
model_ids = get_model_ids()
for model_id in tqdm(model_ids):
meta = get_metadata(model_id)
if meta is None:
continue
for row in parse_metrics_rows(meta):
if row is None:
continue
row["model_id"] = model_id
data.append(row)
return pd.DataFrame.from_records(data)
dataframe = get_data()
selectable_datasets = list(set(dataframe.dataset.tolist()))
st.markdown("# 🤗 Leaderboards")
query_params = st.experimental_get_query_params()
default_dataset = "common_voice"
if "dataset" in query_params:
if len(query_params["dataset"]) > 0 and query_params["dataset"][0] in selectable_datasets:
default_dataset = query_params["dataset"][0]
dataset = st.sidebar.selectbox(
"Dataset",
selectable_datasets,
index=selectable_datasets.index(default_dataset),
)
dataset_df = dataframe[dataframe.dataset == dataset]
dataset_df = dataset_df.dropna(axis="columns", how="all")
metric = st.sidebar.selectbox(
"Metric",
list(filter(lambda column: column not in ("model_id", "dataset"), dataset_df.columns)),
)
dataset_df = dataset_df.filter(["model_id", metric])
dataset_df = dataset_df.dropna()
dataset_df = dataset_df.sort_values(by=metric, ascending=False)
st.markdown(
"Please click on the model's name to be redirected to its model card which includes documentation and examples on how to use it."
)
# display the model ranks
dataset_df = dataset_df.reset_index(drop=True)
dataset_df.index += 1
# turn the model ids into clickable links
dataset_df["model_id"] = dataset_df["model_id"].apply(make_clickable)
table_html = dataset_df.to_html(escape=False)
table_html = table_html.replace("<th>", '<th align="left">') # left-align the headers
st.write(table_html, unsafe_allow_html=True)
st.markdown(
"Want to beat the Leaderboard? Don't see your model here? Simply add the `hf-leaderboards` tag to your model card alongside your evaluation metrics. See [this commit](https://huggingface.co/facebook/wav2vec2-base-960h/commit/88338305603a4d8db25aca96e669beb5f7dc65cb) as an example."
)
|