Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -30,41 +30,48 @@ def get_user_models(hf_username, task):
|
|
30 |
dataset = 'marsyas/gtzan'
|
31 |
case "automatic-speech-recognition":
|
32 |
dataset = 'PolyAI/minds14'
|
|
|
|
|
33 |
case _:
|
34 |
print("Unsupported task")
|
35 |
|
36 |
dataset_specific_models = []
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
41 |
continue
|
42 |
-
|
43 |
-
|
44 |
-
dataset_specific_models.append(model)
|
45 |
-
except: continue
|
46 |
-
return dataset_specific_models
|
47 |
|
48 |
def calculate_best_result(user_models, task):
|
49 |
"""
|
50 |
Calculate the best results of a unit for a given task
|
51 |
:param user_model_ids: models of a user
|
52 |
"""
|
53 |
-
|
54 |
best_model = ""
|
55 |
-
|
56 |
if task == "audio-classification":
|
57 |
best_result = -100
|
58 |
larger_is_better = True
|
59 |
elif task == "automatic-speech-recognition":
|
60 |
best_result = 100
|
61 |
-
larger_is_better = False
|
62 |
|
63 |
for model in user_models:
|
64 |
meta = get_metadata(model)
|
65 |
if meta is None:
|
66 |
-
continue
|
67 |
-
metric = parse_metrics(model, task)
|
68 |
|
69 |
if larger_is_better:
|
70 |
if metric > best_result:
|
@@ -76,7 +83,7 @@ def calculate_best_result(user_models, task):
|
|
76 |
best_model = meta['model-index'][0]["name"]
|
77 |
|
78 |
return best_result, best_model
|
79 |
-
|
80 |
|
81 |
def get_metadata(model_id):
|
82 |
"""
|
@@ -97,19 +104,19 @@ def extract_metric(model_card_content, task):
|
|
97 |
:param model_card_content: model card content
|
98 |
"""
|
99 |
accuracy_pattern = r"Accuracy: (\d+\.\d+)"
|
100 |
-
wer_pattern = r"Wer: (\d+\.\d+)"
|
101 |
-
|
102 |
if task == "audio-classification":
|
103 |
pattern = accuracy_pattern
|
104 |
elif task == "automatic-speech-recognition":
|
105 |
pattern = wer_pattern
|
106 |
-
|
107 |
match = re.search(pattern, model_card_content)
|
108 |
if match:
|
109 |
metric = match.group(1)
|
110 |
return float(metric)
|
111 |
else:
|
112 |
-
return None
|
113 |
|
114 |
|
115 |
def parse_metrics(model, task):
|
@@ -133,16 +140,16 @@ def certification(hf_username):
|
|
133 |
},
|
134 |
{
|
135 |
"unit": "Unit 5: Automatic Speech Recognition",
|
136 |
-
"task": "automatic-speech-recognition",
|
137 |
"baseline_metric": 0.37,
|
138 |
"best_result": 0,
|
139 |
"best_model_id": "",
|
140 |
"passed_": False
|
141 |
},
|
142 |
{
|
143 |
-
"unit": "Unit 6:
|
144 |
-
"task": "
|
145 |
-
"baseline_metric": 0
|
146 |
"best_result": 0,
|
147 |
"best_model_id": "",
|
148 |
"passed_": False
|
@@ -155,7 +162,7 @@ def certification(hf_username):
|
|
155 |
"best_model_id": "",
|
156 |
"passed_": False
|
157 |
},
|
158 |
-
]
|
159 |
|
160 |
for unit in results_certification:
|
161 |
unit["passed"] = pass_emoji(unit["passed_"])
|
@@ -167,31 +174,39 @@ def certification(hf_username):
|
|
167 |
best_result, best_model_id = calculate_best_result(user_ac_models, task = "audio-classification")
|
168 |
unit["best_result"] = best_result
|
169 |
unit["best_model_id"] = best_model_id
|
170 |
-
if unit["best_result"] >= unit["baseline_metric"]:
|
171 |
unit["passed_"] = True
|
172 |
unit["passed"] = pass_emoji(unit["passed_"])
|
173 |
-
except: print("Either no relevant models found, or no metrics in the model card for audio classificaiton")
|
174 |
case "automatic-speech-recognition":
|
175 |
try:
|
176 |
user_asr_models = get_user_models(hf_username, task = "automatic-speech-recognition")
|
177 |
best_result, best_model_id = calculate_best_result(user_asr_models, task = "automatic-speech-recognition")
|
178 |
unit["best_result"] = best_result
|
179 |
unit["best_model_id"] = best_model_id
|
180 |
-
if unit["best_result"] <= unit["baseline_metric"]:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
181 |
unit["passed_"] = True
|
182 |
unit["passed"] = pass_emoji(unit["passed_"])
|
183 |
-
except: print("Either no relevant models found, or no metrics in the model card for automatic speech recognition")
|
184 |
-
case "TBD":
|
185 |
print("Evaluation for this unit is work in progress")
|
186 |
case _:
|
187 |
print("Unknown task")
|
188 |
-
|
189 |
print(results_certification)
|
190 |
-
|
191 |
df = pd.DataFrame(results_certification)
|
192 |
df = df[['passed', 'unit', 'task', 'baseline_metric', 'best_result', 'best_model_id']]
|
193 |
return df
|
194 |
-
|
195 |
with gr.Blocks() as demo:
|
196 |
gr.Markdown(f"""
|
197 |
# π Check your progress in the Audio Course π
|
|
|
30 |
dataset = 'marsyas/gtzan'
|
31 |
case "automatic-speech-recognition":
|
32 |
dataset = 'PolyAI/minds14'
|
33 |
+
case "text-to-speech":
|
34 |
+
dataset = ""
|
35 |
case _:
|
36 |
print("Unsupported task")
|
37 |
|
38 |
dataset_specific_models = []
|
39 |
+
|
40 |
+
if dataset == "":
|
41 |
+
return user_model_ids
|
42 |
+
else:
|
43 |
+
for model in user_model_ids:
|
44 |
+
meta = get_metadata(model)
|
45 |
+
if meta is None:
|
46 |
+
continue
|
47 |
+
try:
|
48 |
+
if meta["datasets"] == [dataset]:
|
49 |
+
dataset_specific_models.append(model)
|
50 |
+
except:
|
51 |
continue
|
52 |
+
return dataset_specific_models
|
53 |
+
|
|
|
|
|
|
|
54 |
|
55 |
def calculate_best_result(user_models, task):
|
56 |
"""
|
57 |
Calculate the best results of a unit for a given task
|
58 |
:param user_model_ids: models of a user
|
59 |
"""
|
60 |
+
|
61 |
best_model = ""
|
62 |
+
|
63 |
if task == "audio-classification":
|
64 |
best_result = -100
|
65 |
larger_is_better = True
|
66 |
elif task == "automatic-speech-recognition":
|
67 |
best_result = 100
|
68 |
+
larger_is_better = False
|
69 |
|
70 |
for model in user_models:
|
71 |
meta = get_metadata(model)
|
72 |
if meta is None:
|
73 |
+
continue
|
74 |
+
metric = parse_metrics(model, task)
|
75 |
|
76 |
if larger_is_better:
|
77 |
if metric > best_result:
|
|
|
83 |
best_model = meta['model-index'][0]["name"]
|
84 |
|
85 |
return best_result, best_model
|
86 |
+
|
87 |
|
88 |
def get_metadata(model_id):
|
89 |
"""
|
|
|
104 |
:param model_card_content: model card content
|
105 |
"""
|
106 |
accuracy_pattern = r"Accuracy: (\d+\.\d+)"
|
107 |
+
wer_pattern = r"Wer: (\d+\.\d+)"
|
108 |
+
|
109 |
if task == "audio-classification":
|
110 |
pattern = accuracy_pattern
|
111 |
elif task == "automatic-speech-recognition":
|
112 |
pattern = wer_pattern
|
113 |
+
|
114 |
match = re.search(pattern, model_card_content)
|
115 |
if match:
|
116 |
metric = match.group(1)
|
117 |
return float(metric)
|
118 |
else:
|
119 |
+
return None
|
120 |
|
121 |
|
122 |
def parse_metrics(model, task):
|
|
|
140 |
},
|
141 |
{
|
142 |
"unit": "Unit 5: Automatic Speech Recognition",
|
143 |
+
"task": "automatic-speech-recognition",
|
144 |
"baseline_metric": 0.37,
|
145 |
"best_result": 0,
|
146 |
"best_model_id": "",
|
147 |
"passed_": False
|
148 |
},
|
149 |
{
|
150 |
+
"unit": "Unit 6: Text-to-Speech",
|
151 |
+
"task": "text-to-speech",
|
152 |
+
"baseline_metric": 0,
|
153 |
"best_result": 0,
|
154 |
"best_model_id": "",
|
155 |
"passed_": False
|
|
|
162 |
"best_model_id": "",
|
163 |
"passed_": False
|
164 |
},
|
165 |
+
]
|
166 |
|
167 |
for unit in results_certification:
|
168 |
unit["passed"] = pass_emoji(unit["passed_"])
|
|
|
174 |
best_result, best_model_id = calculate_best_result(user_ac_models, task = "audio-classification")
|
175 |
unit["best_result"] = best_result
|
176 |
unit["best_model_id"] = best_model_id
|
177 |
+
if unit["best_result"] >= unit["baseline_metric"]:
|
178 |
unit["passed_"] = True
|
179 |
unit["passed"] = pass_emoji(unit["passed_"])
|
180 |
+
except: print("Either no relevant models found, or no metrics in the model card for audio classificaiton")
|
181 |
case "automatic-speech-recognition":
|
182 |
try:
|
183 |
user_asr_models = get_user_models(hf_username, task = "automatic-speech-recognition")
|
184 |
best_result, best_model_id = calculate_best_result(user_asr_models, task = "automatic-speech-recognition")
|
185 |
unit["best_result"] = best_result
|
186 |
unit["best_model_id"] = best_model_id
|
187 |
+
if unit["best_result"] <= unit["baseline_metric"]:
|
188 |
+
unit["passed_"] = True
|
189 |
+
unit["passed"] = pass_emoji(unit["passed_"])
|
190 |
+
except: print("Either no relevant models found, or no metrics in the model card for automatic speech recognition")
|
191 |
+
case "text-to-speech":
|
192 |
+
try:
|
193 |
+
user_tts_models = get_user_models(hf_username, task = "text-to-speech")
|
194 |
+
if user_tts_models:
|
195 |
+
unit["best_result"] = 0
|
196 |
+
unit["best_model_id"] = user_tts_models[0]
|
197 |
unit["passed_"] = True
|
198 |
unit["passed"] = pass_emoji(unit["passed_"])
|
199 |
+
except: print("Either no relevant models found, or no metrics in the model card for automatic speech recognition")
|
|
|
200 |
print("Evaluation for this unit is work in progress")
|
201 |
case _:
|
202 |
print("Unknown task")
|
203 |
+
|
204 |
print(results_certification)
|
205 |
+
|
206 |
df = pd.DataFrame(results_certification)
|
207 |
df = df[['passed', 'unit', 'task', 'baseline_metric', 'best_result', 'best_model_id']]
|
208 |
return df
|
209 |
+
|
210 |
with gr.Blocks() as demo:
|
211 |
gr.Markdown(f"""
|
212 |
# π Check your progress in the Audio Course π
|