Commit
·
5cd28f2
1
Parent(s):
4b6b678
Create new file
Browse files
app.py
ADDED
@@ -0,0 +1,238 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import requests
|
2 |
+
import json
|
3 |
+
import pandas as pd
|
4 |
+
from tqdm.auto import tqdm
|
5 |
+
import streamlit as st
|
6 |
+
from huggingface_hub import HfApi, hf_hub_download
|
7 |
+
from huggingface_hub.repocard import metadata_load
|
8 |
+
|
9 |
+
aliases_lang = {"sv": "sv-SE"}
|
10 |
+
cer_langs = ["ja", "zh-CN", "zh-HK", "zh-TW"]
|
11 |
+
with open("languages.json") as f:
|
12 |
+
lang2name = json.load(f)
|
13 |
+
suggested_datasets = [
|
14 |
+
"librispeech_asr",
|
15 |
+
"mozilla-foundation/common_voice_8_0",
|
16 |
+
"mozilla-foundation/common_voice_7_0",
|
17 |
+
"speech-recognition-community-v2/eval_data",
|
18 |
+
"facebook/multilingual_librispeech"
|
19 |
+
]
|
20 |
+
|
21 |
+
|
22 |
+
def make_clickable(model_name):
|
23 |
+
link = "https://huggingface.co/" + model_name
|
24 |
+
return f'<a target="_blank" href="{link}">{model_name}</a>'
|
25 |
+
|
26 |
+
|
27 |
+
def get_model_ids():
|
28 |
+
api = HfApi()
|
29 |
+
models = api.list_models(filter="hf-asr-leaderboard")
|
30 |
+
model_ids = [x.modelId for x in models]
|
31 |
+
return model_ids
|
32 |
+
|
33 |
+
|
34 |
+
def get_metadata(model_id):
|
35 |
+
try:
|
36 |
+
readme_path = hf_hub_download(model_id, filename="README.md")
|
37 |
+
return metadata_load(readme_path)
|
38 |
+
except:
|
39 |
+
# 404 README.md not found
|
40 |
+
print(f"Model id: {model_id} is not great!")
|
41 |
+
return None
|
42 |
+
|
43 |
+
|
44 |
+
|
45 |
+
def parse_metric_value(value):
|
46 |
+
if isinstance(value, str):
|
47 |
+
"".join(value.split("%"))
|
48 |
+
try:
|
49 |
+
value = float(value)
|
50 |
+
except: # noqa: E722
|
51 |
+
value = None
|
52 |
+
elif isinstance(value, float) and value < 1.1:
|
53 |
+
# assuming that WER is given in 0.xx format
|
54 |
+
value = 100 * value
|
55 |
+
elif isinstance(value, list):
|
56 |
+
if len(value) > 0:
|
57 |
+
value = value[0]
|
58 |
+
else:
|
59 |
+
value = None
|
60 |
+
value = round(value, 2) if value is not None else None
|
61 |
+
return value
|
62 |
+
|
63 |
+
|
64 |
+
def parse_metrics_rows(meta):
|
65 |
+
if "model-index" not in meta or "language" not in meta:
|
66 |
+
return None
|
67 |
+
for result in meta["model-index"][0]["results"]:
|
68 |
+
if "dataset" not in result or "metrics" not in result:
|
69 |
+
continue
|
70 |
+
dataset = result["dataset"]["type"]
|
71 |
+
if "args" in result["dataset"] and "language" in result["dataset"]["args"]:
|
72 |
+
lang = result["dataset"]["args"]["language"]
|
73 |
+
else:
|
74 |
+
lang = meta["language"]
|
75 |
+
lang = lang[0] if isinstance(lang, list) else lang
|
76 |
+
lang = aliases_lang[lang] if lang in aliases_lang else lang
|
77 |
+
config = result["dataset"]["config"] if "config" in result["dataset"] else lang
|
78 |
+
split = result["dataset"]["split"] if "split" in result["dataset"] else None
|
79 |
+
row = {
|
80 |
+
"dataset": dataset,
|
81 |
+
"lang": lang,
|
82 |
+
"config": config,
|
83 |
+
"split": split
|
84 |
+
}
|
85 |
+
for metric in result["metrics"]:
|
86 |
+
type = metric["type"].lower().strip()
|
87 |
+
if type not in ["wer", "cer"]:
|
88 |
+
continue
|
89 |
+
value = parse_metric_value(metric["value"])
|
90 |
+
if value is None:
|
91 |
+
continue
|
92 |
+
if type not in row or value < row[type]:
|
93 |
+
# overwrite the metric if the new value is lower (e.g. with LM)
|
94 |
+
row[type] = value
|
95 |
+
if "wer" in row or "cer" in row:
|
96 |
+
yield row
|
97 |
+
|
98 |
+
|
99 |
+
@st.cache(ttl=600)
|
100 |
+
def get_data():
|
101 |
+
data = []
|
102 |
+
model_ids = get_model_ids()
|
103 |
+
for model_id in tqdm(model_ids):
|
104 |
+
meta = get_metadata(model_id)
|
105 |
+
if meta is None:
|
106 |
+
continue
|
107 |
+
for row in parse_metrics_rows(meta):
|
108 |
+
if row is None:
|
109 |
+
continue
|
110 |
+
row["model_id"] = model_id
|
111 |
+
data.append(row)
|
112 |
+
return pd.DataFrame.from_records(data)
|
113 |
+
|
114 |
+
|
115 |
+
def sort_datasets(datasets):
|
116 |
+
# 1. sort by name
|
117 |
+
datasets = sorted(datasets)
|
118 |
+
# 2. bring the suggested datasets to the top and append the rest
|
119 |
+
datasets = sorted(
|
120 |
+
datasets,
|
121 |
+
key=lambda dataset_id: suggested_datasets.index(dataset_id)
|
122 |
+
if dataset_id in suggested_datasets
|
123 |
+
else len(suggested_datasets),
|
124 |
+
)
|
125 |
+
return datasets
|
126 |
+
|
127 |
+
|
128 |
+
@st.cache(ttl=600)
|
129 |
+
def generate_dataset_info(datasets):
|
130 |
+
msg = """
|
131 |
+
The models have been trained and/or evaluated on the following datasets:
|
132 |
+
"""
|
133 |
+
for dataset_id in datasets:
|
134 |
+
if dataset_id in suggested_datasets:
|
135 |
+
msg += f"* [{dataset_id}](https://hf.co/datasets/{dataset_id}) *(recommended)*\n"
|
136 |
+
else:
|
137 |
+
msg += f"* [{dataset_id}](https://hf.co/datasets/{dataset_id})\n"
|
138 |
+
|
139 |
+
msg = "\n".join([line.strip() for line in msg.split("\n")])
|
140 |
+
return msg
|
141 |
+
|
142 |
+
|
143 |
+
dataframe = get_data()
|
144 |
+
dataframe = dataframe.fillna("")
|
145 |
+
|
146 |
+
st.sidebar.image("logo.png", width=200)
|
147 |
+
|
148 |
+
st.markdown("# The 🤗 Speech Bench")
|
149 |
+
|
150 |
+
st.markdown(
|
151 |
+
f"This is a leaderboard of **{dataframe['model_id'].nunique()}** speech recognition models "
|
152 |
+
f"and **{dataframe['dataset'].nunique()}** datasets.\n\n"
|
153 |
+
"⬅ Please select the language you want to find a model for from the dropdown on the left."
|
154 |
+
)
|
155 |
+
|
156 |
+
lang = st.sidebar.selectbox(
|
157 |
+
"Language",
|
158 |
+
sorted(dataframe["lang"].unique(), key=lambda key: lang2name.get(key, key)),
|
159 |
+
format_func=lambda key: lang2name.get(key, key),
|
160 |
+
index=0,
|
161 |
+
)
|
162 |
+
lang_df = dataframe[dataframe.lang == lang]
|
163 |
+
|
164 |
+
sorted_datasets = sort_datasets(lang_df["dataset"].unique())
|
165 |
+
|
166 |
+
lang_name = lang2name[lang] if lang in lang2name else ""
|
167 |
+
num_models = len(lang_df["model_id"].unique())
|
168 |
+
num_datasets = len(lang_df["dataset"].unique())
|
169 |
+
text = f"""
|
170 |
+
For the `{lang}` ({lang_name}) language, there are currently `{num_models}` model(s)
|
171 |
+
trained on `{num_datasets}` dataset(s) available for `automatic-speech-recognition`.
|
172 |
+
"""
|
173 |
+
st.markdown(text)
|
174 |
+
|
175 |
+
st.sidebar.markdown("""
|
176 |
+
Choose the dataset that is most relevant to your task and select it from the dropdown below:
|
177 |
+
""")
|
178 |
+
|
179 |
+
dataset = st.sidebar.selectbox(
|
180 |
+
"Dataset",
|
181 |
+
sorted_datasets,
|
182 |
+
index=0,
|
183 |
+
)
|
184 |
+
dataset_df = lang_df[lang_df.dataset == dataset]
|
185 |
+
|
186 |
+
text = generate_dataset_info(sorted_datasets)
|
187 |
+
st.sidebar.markdown(text)
|
188 |
+
|
189 |
+
# sort by WER or CER depending on the language
|
190 |
+
metric_col = "cer" if lang in cer_langs else "wer"
|
191 |
+
if dataset_df["config"].nunique() > 1:
|
192 |
+
# if there are more than one dataset config
|
193 |
+
dataset_df = dataset_df[["model_id", "config", metric_col]]
|
194 |
+
dataset_df = dataset_df.pivot_table(index=['model_id'], columns=["config"], values=[metric_col])
|
195 |
+
dataset_df = dataset_df.reset_index(level=0)
|
196 |
+
else:
|
197 |
+
dataset_df = dataset_df[["model_id", metric_col]]
|
198 |
+
dataset_df.sort_values(dataset_df.columns[-1], inplace=True)
|
199 |
+
dataset_df = dataset_df.fillna("")
|
200 |
+
|
201 |
+
dataset_df.rename(
|
202 |
+
columns={
|
203 |
+
"model_id": "Model",
|
204 |
+
"wer": "WER (lower is better)",
|
205 |
+
"cer": "CER (lower is better)",
|
206 |
+
},
|
207 |
+
inplace=True,
|
208 |
+
)
|
209 |
+
|
210 |
+
st.markdown(
|
211 |
+
"Please click on the model's name to be redirected to its model card which includes documentation and examples on how to use it."
|
212 |
+
)
|
213 |
+
|
214 |
+
# display the model ranks
|
215 |
+
dataset_df = dataset_df.reset_index(drop=True)
|
216 |
+
dataset_df.index += 1
|
217 |
+
|
218 |
+
# turn the model ids into clickable links
|
219 |
+
dataset_df["Model"] = dataset_df["Model"].apply(make_clickable)
|
220 |
+
|
221 |
+
table_html = dataset_df.to_html(escape=False)
|
222 |
+
table_html = table_html.replace("<th>", '<th align="left">') # left-align the headers
|
223 |
+
st.write(table_html, unsafe_allow_html=True)
|
224 |
+
|
225 |
+
if lang in cer_langs:
|
226 |
+
st.markdown(
|
227 |
+
"---\n\* **CER** is [Char Error Rate](https://huggingface.co/metrics/cer)"
|
228 |
+
)
|
229 |
+
else:
|
230 |
+
st.markdown(
|
231 |
+
"---\n\* **WER** is [Word Error Rate](https://huggingface.co/metrics/wer)"
|
232 |
+
)
|
233 |
+
|
234 |
+
st.markdown(
|
235 |
+
"Want to beat the Leaderboard? Don't see your speech recognition model show up here? "
|
236 |
+
"Simply add the `hf-asr-leaderboard` tag to your model card alongside your evaluation metrics. "
|
237 |
+
"Try our [Metrics Editor](https://huggingface.co/spaces/huggingface/speech-bench-metrics-editor) to get started!"
|
238 |
+
)
|