|
import requests |
|
import json |
|
import pandas as pd |
|
from tqdm.auto import tqdm |
|
import streamlit as st |
|
from pandas import read_csv |
|
import os |
|
from huggingface_hub import HfApi, hf_hub_download |
|
from huggingface_hub.repocard import metadata_load |
|
import jiwer |
|
import datetime |
|
from huggingface_hub import Repository |
|
|
|
REFERENCE_NAME = "references" |
|
SUBMISSION_NAME = "submissions" |
|
|
|
REFERENCE_URL = os.path.join("https://huggingface.co/datasets/esc-bench", REFERENCE_NAME) |
|
SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/esc-bench", SUBMISSION_NAME) |
|
|
|
TEST_SETS = ["librispeech-clean", "librispeech-other", "common-voice-9", "vox-populi", "ted-lium", "giga-speech", "spgi-speech", "earnings-22", "ami"] |
|
EXPECTED_TEST_FILES = [f + ".txt" for f in TEST_SETS] |
|
OPTIONAL_TEST_SETS = ["switch-board", "call-home", "chime-4"] |
|
|
|
CSV_RESULTS_FILE = os.path.join(SUBMISSION_NAME, "results.csv") |
|
|
|
|
|
HF_TOKEN = os.environ.get("HF_TOKEN") |
|
|
|
|
|
def compute_wer(pred_file, ref_file): |
|
with open(pred_file, "r", encoding="utf-8") as pred, open(ref_file, "r", encoding="utf-8") as ref: |
|
pred_lines = [line.strip() for line in pred.readlines()] |
|
ref_lines = [line.strip() for line in ref.readlines()] |
|
|
|
wer = jiwer.wer(ref_lines, pred_lines) |
|
return wer |
|
|
|
|
|
reference_repo = Repository(local_dir="references", clone_from=REFERENCE_URL, use_auth_token=HF_TOKEN) |
|
submission_repo = Repository(local_dir="submissions", clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN) |
|
|
|
all_submissions = [folder for folder in os.listdir(SUBMISSION_NAME) if os.path.isdir(os.path.join(SUBMISSION_NAME, folder)) and folder != ".git"] |
|
|
|
all_results = read_csv(CSV_RESULTS_FILE) |
|
evaluated_submissions = all_results["name"].values.tolist() |
|
|
|
non_evaluated_submissions = set(all_submissions) - set(evaluated_submissions) |
|
if len(non_evaluated_submissions) > 0: |
|
for submission in non_evaluated_submissions: |
|
print(f"Evaluate {submission}") |
|
results = {"name": submission} |
|
submitted_files = os.listdir(os.path.join(SUBMISSION_NAME, submission)) |
|
|
|
submitted_files = [f for f in submitted_files if f in EXPECTED_TEST_FILES] |
|
|
|
if sorted(EXPECTED_TEST_FILES) != sorted(submitted_files): |
|
raise ValueError(f"{', '.join(submitted_files)} were submitted, but expected {', '.join(EXPECTED_TEST_FILES)}") |
|
|
|
for file in submitted_files: |
|
ref_file = os.path.join(REFERENCE_NAME, file) |
|
pred_file = os.path.join(SUBMISSION_NAME, submission, file) |
|
|
|
wer = compute_wer(pred_file, ref_file) |
|
results[file.split(".")[0]] = str(wer) |
|
|
|
wer_values = [float(results[t]) for t in TEST_SETS] |
|
all_wer = sum(wer_values) / len(wer_values) |
|
|
|
results["esc-score"] = all_wer |
|
all_results = all_results.append(results, ignore_index=True) |
|
|
|
|
|
all_results.to_csv(CSV_RESULTS_FILE) |
|
|
|
commit_url = reference_repo.push_to_hub() |
|
print(commit_url) |
|
|
|
st.table(all_results) |
|
|
