|
import gradio as gr |
|
import pandas as pd |
|
import json |
|
import os |
|
from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message |
|
from datetime import datetime, timezone |
|
from pathlib import Path |
|
from request_api import make_get_request, make_post_request |
|
from io import BytesIO, StringIO |
|
LEADERBOARD_PATH = "/home/Bhattacharya/ism_leaderboard/files/leaderboard" |
|
|
|
DIR_OUTPUT_REQUESTS = Path("requested_models") |
|
EVAL_REQUESTS_PATH = Path("eval_requests") |
|
|
|
|
|
|
|
|
|
|
|
banner_url = "https://huggingface.co/spaces/debayan/ism_2023w/resolve/main/logo_leaderboard.png" |
|
BANNER = f'<div style="display: flex; justify-content: space-around;"><img src="{banner_url}" alt="Banner" style="width: 40vw; min-width: 300px; max-width: 600px;"> </div>' |
|
|
|
TITLE = "<html> <head> <style> h1 {text-align: center;} </style> </head> <body> <h1> 🤗 Open Automatic Speech Recognition Leaderboard </b> </body> </html>" |
|
|
|
INTRODUCTION_TEXT = "🏆 The ISM2023w Leaderboard ranks and evaluates models \ |
|
on the Hugging Face Spaces. \ |
|
\nWe report the Precision, Recall, Accuracy, Weighted [F1 Score](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html)\n" |
|
|
|
|
|
CSV_TEXT = "<p>Please use this link to upload your csvs and check leaderboards!</p> <a href=\"https://a37fcb50ccc3b4a060.gradio.live\" >https://a37fcb50ccc3b4a060.gradio.live</a>" |
|
|
|
|
|
|
|
METRICS_TAB_TEXT = """ |
|
Here you will find details about the multi-class classification metrics and datasets reported in our leaderboard. |
|
## Metrics |
|
🎯 Word Error Rate (WER) and Real-Time Factor (RTF) are popular metrics for evaluating the accuracy of speech recognition |
|
models by estimating how accurate the predictions from the models are and how fast they are returned. We explain them each |
|
below. |
|
""" |
|
|
|
|
|
LAST_UPDATED = "Sep 28th 2023" |
|
|
|
|
|
|
|
def create_html_page(title, content): |
|
|
|
html_page = f""" |
|
<html> |
|
<head> |
|
<title>{title}</title> |
|
</head> |
|
<body> |
|
{content} |
|
</body> |
|
</html> |
|
""" |
|
|
|
return html_page |
|
|
|
def fetch_leaderboard(phase=1): |
|
|
|
""" |
|
Fetch the leaderboard from the local disk |
|
""" |
|
params = {"phase": phase} |
|
|
|
leaderboard_df = make_get_request("fetch-leaderboard", params) |
|
csv_data = leaderboard_df.content.decode('utf-8') |
|
|
|
print(csv_data) |
|
|
|
|
|
leaderboard_df = pd.read_csv(StringIO(csv_data)) |
|
|
|
leaderboard_df = leaderboard_df.dropna() |
|
|
|
return leaderboard_df |
|
|
|
def process_csv_text(temp_file): |
|
if isinstance(temp_file, str): |
|
|
|
df = pd.read_csv(StringIO(temp_file)) |
|
|
|
else: |
|
try: |
|
df = pd.read_csv(temp_file.name) |
|
except: |
|
return None |
|
|
|
return df |
|
|
|
def app(team_name, password, csv_file): |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
returned_team_name_json = make_get_request("get-team-name", {"password": password}).json() |
|
|
|
if "error" in returned_team_name_json: |
|
|
|
return create_html_page("Error", "Invalid team name or password.") |
|
|
|
else: |
|
returned_team_name = returned_team_name_json["team_name"] |
|
|
|
if team_name != returned_team_name: |
|
return create_html_page("Error", "Invalid team name or password.") |
|
|
|
else: |
|
print("team name and password are correct") |
|
|
|
|
|
|
|
|
|
|
|
|
|
df = process_csv_text(csv_file) |
|
if df is None: |
|
return create_html_page("Error", "Invalid CSV file format.") |
|
csv_data = df.to_csv(index=False) |
|
|
|
|
|
upload_csv_response = make_post_request("upload", {"password": password}, data = csv_data) |
|
|
|
|
|
if "error" in upload_csv_response.json(): |
|
|
|
|
|
return create_html_page("Error", upload_csv_response.json()["error"] ) |
|
|
|
else: |
|
|
|
return create_html_page("Success", upload_csv_response.json()["message"]) |
|
|
|
|
|
def formatter(x): |
|
|
|
x = round(x, 2) |
|
return x |
|
|
|
def format_leaderboard(df,sort_by="F1 Score"): |
|
|
|
|
|
for col in df.columns: |
|
if col == "Team Name": |
|
|
|
continue |
|
|
|
elif col == "Team Members": |
|
|
|
continue |
|
else: |
|
|
|
if df[col].dtype == "float64" or df[col].dtype == "int64": |
|
df[col] = df[col].apply(formatter) |
|
df.sort_values(by=sort_by, inplace=True, ascending=False) |
|
return df |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
COLS = [c.name for c in fields(AutoEvalColumn)] |
|
TYPES = [c.type for c in fields(AutoEvalColumn)] |
|
|
|
with gr.Blocks() as demo: |
|
leaderboard_table_1 = None |
|
leaderboard_table_2 = None |
|
|
|
def button_clicked_phase_1(inputs): |
|
|
|
leaderboard_df_1 = format_leaderboard(fetch_leaderboard(phase=1)) |
|
|
|
|
|
leaderboard_table_1 = gr.components.Dataframe( |
|
value=leaderboard_df_1, |
|
datatype=TYPES, |
|
max_rows=None, |
|
elem_id="leaderboard-table-1", |
|
interactive=False, |
|
visible=True, |
|
|
|
) |
|
|
|
return leaderboard_table_1 |
|
|
|
|
|
def button_clicked_phase_2(): |
|
|
|
leaderboard_df_2 = format_leaderboard(fetch_leaderboard(phase=2)) |
|
|
|
|
|
leaderboard_table_2 = gr.components.Dataframe( |
|
value=leaderboard_df_2, |
|
datatype=TYPES, |
|
max_rows=None, |
|
elem_id="leaderboard-table-2", |
|
interactive=False, |
|
visible=True, |
|
|
|
) |
|
|
|
return leaderboard_table_2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
gr.HTML(BANNER, elem_id="banner") |
|
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") |
|
gr.HTML(CSV_TEXT, elem_classes="markdown-text") |
|
|
|
|
|
""" |
|
with gr.Tabs(elem_classes="tab-buttons") as tabs: |
|
with gr.TabItem("🏅 Leaderboard Phase 1", elem_id="od-benchmark-tab-table-1", id=0): |
|
|
|
leaderboard_table_1 = gr.components.Dataframe( |
|
value=leaderboard_df_1, |
|
datatype=TYPES, |
|
max_rows=None, |
|
elem_id="leaderboard-table-1", |
|
interactive=False, |
|
visible=True, |
|
|
|
) |
|
button = gr.Button("Refresh Leaderboard",interactive=True) |
|
button.click(fn=button_clicked_phase_1,outputs=[leaderboard_table_1]) |
|
|
|
with gr.TabItem("🏅 Leaderboard Phase 2", elem_id="od-benchmark-tab-table-2", id=1): |
|
|
|
|
|
leaderboard_table_2 = gr.components.Dataframe( |
|
value=leaderboard_df_2, |
|
datatype=TYPES, |
|
max_rows=None, |
|
elem_id="leaderboard-table-2", |
|
interactive=False, |
|
visible=True, |
|
|
|
) |
|
button = gr.Button("Refresh Leaderboard",interactive=True) |
|
button.click(fn=button_clicked_phase_2,outputs=[leaderboard_table_2]) |
|
|
|
|
|
with gr.TabItem("📈 Upload CSV", elem_id="od-benchmark-tab-table", id=2): |
|
|
|
gr.Interface( |
|
fn=app, |
|
title='CSV Upload and Leaderboard', |
|
description='Upload a CSV file and see your team\'s rank on the leaderboard.', |
|
layout='vertical', |
|
theme='compact', |
|
inputs=[ |
|
gr.Textbox(label='Team Name'), |
|
gr.Textbox(label='Password'), |
|
gr.File(file_count= "single", file_types =[".txt",".csv"] ,label='CSV File') # only allow csv files |
|
], |
|
outputs=gr.outputs.HTML() |
|
) |
|
""" |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text") |
|
|
|
demo.launch(debug=True,share=True) |
|
|
|
|
|
|
