File size: 2,988 Bytes
5cd28f2 0806a18 5cd28f2 0806a18 ae20e1b 0806a18 5cd28f2 0806a18 ae20e1b 0806a18 ae20e1b 0806a18 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
import requests
import json
import pandas as pd
from tqdm.auto import tqdm
import streamlit as st
from pandas import read_csv
import os
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
import jiwer
import datetime
from huggingface_hub import Repository
REFERENCE_NAME = "references"
SUBMISSION_NAME = "submissions"
REFERENCE_URL = os.path.join("https://huggingface.co/datasets/esc-bench", REFERENCE_NAME)
SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/esc-bench", SUBMISSION_NAME)
TEST_SETS = ["librispeech-clean", "librispeech-other", "common-voice-9", "vox-populi", "ted-lium", "giga-speech", "spgi-speech", "earnings-22", "ami"]
EXPECTED_TEST_FILES = [f + ".txt" for f in TEST_SETS]
OPTIONAL_TEST_SETS = ["switch-board", "call-home", "chime-4"]
CSV_RESULTS_FILE = os.path.join(SUBMISSION_NAME, "results.csv")
HF_TOKEN = os.environ.get("HF_TOKEN")
def compute_wer(pred_file, ref_file):
with open(pred_file, "r", encoding="utf-8") as pred, open(ref_file, "r", encoding="utf-8") as ref:
pred_lines = [line.strip() for line in pred.readlines()]
ref_lines = [line.strip() for line in ref.readlines()]
wer = jiwer.wer(ref_lines, pred_lines)
return wer
reference_repo = Repository(local_dir="references", clone_from=REFERENCE_URL, use_auth_token=HF_TOKEN)
submission_repo = Repository(local_dir="submissions", clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN)
all_submissions = [folder for folder in os.listdir(SUBMISSION_NAME) if os.path.isdir(os.path.join(SUBMISSION_NAME, folder)) and folder != ".git"]
all_results = read_csv(CSV_RESULTS_FILE)
evaluated_submissions = all_results["name"].values.tolist()
non_evaluated_submissions = set(all_submissions) - set(evaluated_submissions)
if len(non_evaluated_submissions) > 0:
for submission in non_evaluated_submissions:
print(f"Evaluate {submission}")
results = {"name": submission}
submitted_files = os.listdir(os.path.join(SUBMISSION_NAME, submission))
submitted_files = [f for f in submitted_files if f in EXPECTED_TEST_FILES]
if sorted(EXPECTED_TEST_FILES) != sorted(submitted_files):
raise ValueError(f"{', '.join(submitted_files)} were submitted, but expected {', '.join(EXPECTED_TEST_FILES)}")
for file in submitted_files:
ref_file = os.path.join(REFERENCE_NAME, file)
pred_file = os.path.join(SUBMISSION_NAME, submission, file)
wer = compute_wer(pred_file, ref_file)
results[file.split(".")[0]] = str(wer)
wer_values = [float(results[t]) for t in TEST_SETS]
all_wer = sum(wer_values) / len(wer_values)
results["esc-score"] = all_wer
all_results = all_results.append(results, ignore_index=True)
# save and upload new evaluated results
all_results.to_csv(CSV_RESULTS_FILE)
commit_url = reference_repo.push_to_hub()
print(commit_url)
st.table(all_results)
|