patrickvonplaten commited on
Commit
5cd28f2
·
1 Parent(s): 4b6b678

Create new file

Browse files
Files changed (1) hide show
  1. app.py +238 -0
app.py ADDED
@@ -0,0 +1,238 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import requests
2
+ import json
3
+ import pandas as pd
4
+ from tqdm.auto import tqdm
5
+ import streamlit as st
6
+ from huggingface_hub import HfApi, hf_hub_download
7
+ from huggingface_hub.repocard import metadata_load
8
+
9
+ aliases_lang = {"sv": "sv-SE"}
10
+ cer_langs = ["ja", "zh-CN", "zh-HK", "zh-TW"]
11
+ with open("languages.json") as f:
12
+ lang2name = json.load(f)
13
+ suggested_datasets = [
14
+ "librispeech_asr",
15
+ "mozilla-foundation/common_voice_8_0",
16
+ "mozilla-foundation/common_voice_7_0",
17
+ "speech-recognition-community-v2/eval_data",
18
+ "facebook/multilingual_librispeech"
19
+ ]
20
+
21
+
22
+ def make_clickable(model_name):
23
+ link = "https://huggingface.co/" + model_name
24
+ return f'<a target="_blank" href="{link}">{model_name}</a>'
25
+
26
+
27
+ def get_model_ids():
28
+ api = HfApi()
29
+ models = api.list_models(filter="hf-asr-leaderboard")
30
+ model_ids = [x.modelId for x in models]
31
+ return model_ids
32
+
33
+
34
+ def get_metadata(model_id):
35
+ try:
36
+ readme_path = hf_hub_download(model_id, filename="README.md")
37
+ return metadata_load(readme_path)
38
+ except:
39
+ # 404 README.md not found
40
+ print(f"Model id: {model_id} is not great!")
41
+ return None
42
+
43
+
44
+
45
+ def parse_metric_value(value):
46
+ if isinstance(value, str):
47
+ "".join(value.split("%"))
48
+ try:
49
+ value = float(value)
50
+ except: # noqa: E722
51
+ value = None
52
+ elif isinstance(value, float) and value < 1.1:
53
+ # assuming that WER is given in 0.xx format
54
+ value = 100 * value
55
+ elif isinstance(value, list):
56
+ if len(value) > 0:
57
+ value = value[0]
58
+ else:
59
+ value = None
60
+ value = round(value, 2) if value is not None else None
61
+ return value
62
+
63
+
64
+ def parse_metrics_rows(meta):
65
+ if "model-index" not in meta or "language" not in meta:
66
+ return None
67
+ for result in meta["model-index"][0]["results"]:
68
+ if "dataset" not in result or "metrics" not in result:
69
+ continue
70
+ dataset = result["dataset"]["type"]
71
+ if "args" in result["dataset"] and "language" in result["dataset"]["args"]:
72
+ lang = result["dataset"]["args"]["language"]
73
+ else:
74
+ lang = meta["language"]
75
+ lang = lang[0] if isinstance(lang, list) else lang
76
+ lang = aliases_lang[lang] if lang in aliases_lang else lang
77
+ config = result["dataset"]["config"] if "config" in result["dataset"] else lang
78
+ split = result["dataset"]["split"] if "split" in result["dataset"] else None
79
+ row = {
80
+ "dataset": dataset,
81
+ "lang": lang,
82
+ "config": config,
83
+ "split": split
84
+ }
85
+ for metric in result["metrics"]:
86
+ type = metric["type"].lower().strip()
87
+ if type not in ["wer", "cer"]:
88
+ continue
89
+ value = parse_metric_value(metric["value"])
90
+ if value is None:
91
+ continue
92
+ if type not in row or value < row[type]:
93
+ # overwrite the metric if the new value is lower (e.g. with LM)
94
+ row[type] = value
95
+ if "wer" in row or "cer" in row:
96
+ yield row
97
+
98
+
99
+ @st.cache(ttl=600)
100
+ def get_data():
101
+ data = []
102
+ model_ids = get_model_ids()
103
+ for model_id in tqdm(model_ids):
104
+ meta = get_metadata(model_id)
105
+ if meta is None:
106
+ continue
107
+ for row in parse_metrics_rows(meta):
108
+ if row is None:
109
+ continue
110
+ row["model_id"] = model_id
111
+ data.append(row)
112
+ return pd.DataFrame.from_records(data)
113
+
114
+
115
+ def sort_datasets(datasets):
116
+ # 1. sort by name
117
+ datasets = sorted(datasets)
118
+ # 2. bring the suggested datasets to the top and append the rest
119
+ datasets = sorted(
120
+ datasets,
121
+ key=lambda dataset_id: suggested_datasets.index(dataset_id)
122
+ if dataset_id in suggested_datasets
123
+ else len(suggested_datasets),
124
+ )
125
+ return datasets
126
+
127
+
128
+ @st.cache(ttl=600)
129
+ def generate_dataset_info(datasets):
130
+ msg = """
131
+ The models have been trained and/or evaluated on the following datasets:
132
+ """
133
+ for dataset_id in datasets:
134
+ if dataset_id in suggested_datasets:
135
+ msg += f"* [{dataset_id}](https://hf.co/datasets/{dataset_id}) *(recommended)*\n"
136
+ else:
137
+ msg += f"* [{dataset_id}](https://hf.co/datasets/{dataset_id})\n"
138
+
139
+ msg = "\n".join([line.strip() for line in msg.split("\n")])
140
+ return msg
141
+
142
+
143
+ dataframe = get_data()
144
+ dataframe = dataframe.fillna("")
145
+
146
+ st.sidebar.image("logo.png", width=200)
147
+
148
+ st.markdown("# The 🤗 Speech Bench")
149
+
150
+ st.markdown(
151
+ f"This is a leaderboard of **{dataframe['model_id'].nunique()}** speech recognition models "
152
+ f"and **{dataframe['dataset'].nunique()}** datasets.\n\n"
153
+ "⬅ Please select the language you want to find a model for from the dropdown on the left."
154
+ )
155
+
156
+ lang = st.sidebar.selectbox(
157
+ "Language",
158
+ sorted(dataframe["lang"].unique(), key=lambda key: lang2name.get(key, key)),
159
+ format_func=lambda key: lang2name.get(key, key),
160
+ index=0,
161
+ )
162
+ lang_df = dataframe[dataframe.lang == lang]
163
+
164
+ sorted_datasets = sort_datasets(lang_df["dataset"].unique())
165
+
166
+ lang_name = lang2name[lang] if lang in lang2name else ""
167
+ num_models = len(lang_df["model_id"].unique())
168
+ num_datasets = len(lang_df["dataset"].unique())
169
+ text = f"""
170
+ For the `{lang}` ({lang_name}) language, there are currently `{num_models}` model(s)
171
+ trained on `{num_datasets}` dataset(s) available for `automatic-speech-recognition`.
172
+ """
173
+ st.markdown(text)
174
+
175
+ st.sidebar.markdown("""
176
+ Choose the dataset that is most relevant to your task and select it from the dropdown below:
177
+ """)
178
+
179
+ dataset = st.sidebar.selectbox(
180
+ "Dataset",
181
+ sorted_datasets,
182
+ index=0,
183
+ )
184
+ dataset_df = lang_df[lang_df.dataset == dataset]
185
+
186
+ text = generate_dataset_info(sorted_datasets)
187
+ st.sidebar.markdown(text)
188
+
189
+ # sort by WER or CER depending on the language
190
+ metric_col = "cer" if lang in cer_langs else "wer"
191
+ if dataset_df["config"].nunique() > 1:
192
+ # if there are more than one dataset config
193
+ dataset_df = dataset_df[["model_id", "config", metric_col]]
194
+ dataset_df = dataset_df.pivot_table(index=['model_id'], columns=["config"], values=[metric_col])
195
+ dataset_df = dataset_df.reset_index(level=0)
196
+ else:
197
+ dataset_df = dataset_df[["model_id", metric_col]]
198
+ dataset_df.sort_values(dataset_df.columns[-1], inplace=True)
199
+ dataset_df = dataset_df.fillna("")
200
+
201
+ dataset_df.rename(
202
+ columns={
203
+ "model_id": "Model",
204
+ "wer": "WER (lower is better)",
205
+ "cer": "CER (lower is better)",
206
+ },
207
+ inplace=True,
208
+ )
209
+
210
+ st.markdown(
211
+ "Please click on the model's name to be redirected to its model card which includes documentation and examples on how to use it."
212
+ )
213
+
214
+ # display the model ranks
215
+ dataset_df = dataset_df.reset_index(drop=True)
216
+ dataset_df.index += 1
217
+
218
+ # turn the model ids into clickable links
219
+ dataset_df["Model"] = dataset_df["Model"].apply(make_clickable)
220
+
221
+ table_html = dataset_df.to_html(escape=False)
222
+ table_html = table_html.replace("<th>", '<th align="left">') # left-align the headers
223
+ st.write(table_html, unsafe_allow_html=True)
224
+
225
+ if lang in cer_langs:
226
+ st.markdown(
227
+ "---\n\* **CER** is [Char Error Rate](https://huggingface.co/metrics/cer)"
228
+ )
229
+ else:
230
+ st.markdown(
231
+ "---\n\* **WER** is [Word Error Rate](https://huggingface.co/metrics/wer)"
232
+ )
233
+
234
+ st.markdown(
235
+ "Want to beat the Leaderboard? Don't see your speech recognition model show up here? "
236
+ "Simply add the `hf-asr-leaderboard` tag to your model card alongside your evaluation metrics. "
237
+ "Try our [Metrics Editor](https://huggingface.co/spaces/huggingface/speech-bench-metrics-editor) to get started!"
238
+ )