import gradio as gr import random import os import shutil import pandas as pd import sqlite3 from datasets import load_dataset import threading import time from huggingface_hub import HfApi DESCR = """ # TTS Arena Vote on different speech synthesis models! """.strip() INSTR = """ ## Instructions * Listen to two anonymous models * Vote on which one is more natural and realistic * If there's a tie, click Skip *IMPORTANT: Do not only rank the outputs based on naturalness. Also rank based on intelligibility (can you actually tell what they're saying?) and other factors (does it sound like a human?).* **When you're ready to begin, click the Start button below!** The model names will be revealed once you vote. """.strip() request = '' if os.getenv('HF_ID'): request = f""" ### Request Model Please fill out [this form](https://huggingface.co/spaces/{os.getenv('HF_ID')}/discussions/new?title=%5BModel+Request%5D+&description=%23%23%20Model%20Request%0A%0A%2A%2AModel%20website%2Fpaper%20%28if%20applicable%29%2A%2A%3A%0A%2A%2AModel%20available%20on%2A%2A%3A%20%28coqui%7CHF%20pipeline%7Ccustom%20code%29%0A%2A%2AWhy%20do%20you%20want%20this%20model%20added%3F%2A%2A%0A%2A%2AComments%3A%2A%2A) to request a model. """ ABOUT = f""" ## About TTS Arena is a project created to evaluate leading speech synthesis models. It is inspired by the [Chatbot Arena](https://chat.lmsys.org/) by LMSys. {request} """.strip() LDESC = """ ## Leaderboard A list of the models, based on how highly they are ranked! """.strip() dataset = load_dataset("ttseval/tts-arena", token=os.getenv('HF_TOKEN')) theme = gr.themes.Base( font=[gr.themes.GoogleFont('Libre Franklin'), gr.themes.GoogleFont('Public Sans'), 'system-ui', 'sans-serif'], ) model_names = { 'styletts2': 'StyleTTS 2', 'tacotron': 'Tacotron', 'tacotronph': 'Tacotron Phoneme', 'tacotrondca': 'Tacotron DCA', 'speedyspeech': 'Speedy Speech', 'overflow': 'Overflow TTS', 'vits': 'VITS', 'vitsneon': 'VITS Neon', 'neuralhmm': 'Neural HMM', 'glow': 'Glow TTS', 'fastpitch': 'FastPitch', 'jenny': 'Jenny', 'tortoise': 'Tortoise TTS', 'xtts2': 'XTTSv2', 'xtts': 'XTTS', 'elevenlabs': 'ElevenLabs', 'speecht5': 'SpeechT5', } def get_random_split(existing_split=None): choice = random.choice(list(dataset.keys())) if existing_split and choice == existing_split: return get_random_split(choice) else: return choice def get_db(): return sqlite3.connect('database.db') def create_db(): conn = get_db() cursor = conn.cursor() cursor.execute(''' CREATE TABLE IF NOT EXISTS model ( name TEXT UNIQUE, upvote INTEGER, downvote INTEGER ); ''') def get_data(): conn = get_db() cursor = conn.cursor() cursor.execute('SELECT name, upvote, downvote FROM model') data = cursor.fetchall() df = pd.DataFrame(data, columns=['name', 'upvote', 'downvote']) df['name'] = df['name'].replace(model_names) df['votes'] = df['upvote'] + df['downvote'] # df['score'] = round((df['upvote'] / df['votes']) * 100, 2) # Percentage score ## ELO SCORE df['score'] = 1200 for i in range(len(df)): for j in range(len(df)): if i != j: expected_a = 1 / (1 + 10 ** ((df['score'][j] - df['score'][i]) / 400)) expected_b = 1 / (1 + 10 ** ((df['score'][i] - df['score'][j]) / 400)) actual_a = df['upvote'][i] / df['votes'][i] actual_b = df['upvote'][j] / df['votes'][j] df.at[i, 'score'] += 32 * (actual_a - expected_a) df.at[j, 'score'] += 32 * (actual_b - expected_b) if df['votes'][j] < 3: df.at[j, 'score'] -= (3 - df['votes'][j]) * 5 df['score'] = round(df['score']) ## ELO SCORE df = df.sort_values(by='score', ascending=False) # df = df[['name', 'score', 'upvote', 'votes']] df = df[['name', 'score', 'votes']] return df def get_random_splits(): choice1 = get_random_split() choice2 = get_random_split(choice1) return (choice1, choice2) def upvote_model(model): conn = get_db() cursor = conn.cursor() cursor.execute('UPDATE model SET upvote = upvote + 1 WHERE name = ?', (model,)) if cursor.rowcount == 0: cursor.execute('INSERT OR REPLACE INTO model (name, upvote, downvote) VALUES (?, 1, 0)', (model,)) conn.commit() cursor.close() def downvote_model(model): conn = get_db() cursor = conn.cursor() cursor.execute('UPDATE model SET downvote = downvote + 1 WHERE name = ?', (model,)) if cursor.rowcount == 0: cursor.execute('INSERT OR REPLACE INTO model (name, upvote, downvote) VALUES (?, 0, 1)', (model,)) conn.commit() cursor.close() def a_is_better(model1, model2): if model1 and model2: upvote_model(model1) downvote_model(model2) return reload(model1, model2) def b_is_better(model1, model2): if model1 and model2: upvote_model(model2) downvote_model(model1) return reload(model1, model2) def both_bad(model1, model2): if model1 and model2: downvote_model(model1) downvote_model(model2) return reload(model1, model2) def both_good(model1, model2): if model1 and model2: upvote_model(model1) upvote_model(model2) return reload(model1, model2) def reload(chosenmodel1=None, chosenmodel2=None): # Select random splits split1, split2 = get_random_splits() d1, d2 = (dataset[split1], dataset[split2]) choice1, choice2 = (d1.shuffle()[0]['audio'], d2.shuffle()[0]['audio']) if chosenmodel1 in model_names: chosenmodel1 = model_names[chosenmodel1] if chosenmodel2 in model_names: chosenmodel2 = model_names[chosenmodel2] out = [ (choice1['sampling_rate'], choice1['array']), (choice2['sampling_rate'], choice2['array']), split1, split2 ] if chosenmodel1: out.append(f'This model was {chosenmodel1}') if chosenmodel2: out.append(f'This model was {chosenmodel2}') return out with gr.Blocks() as leaderboard: gr.Markdown(LDESC) # df = gr.Dataframe(interactive=False, value=get_data()) df = gr.Dataframe(interactive=False, min_width=0, wrap=True, column_widths=[200, 50, 50]) leaderboard.load(get_data, outputs=[df]) with gr.Blocks() as vote: gr.Markdown(INSTR) with gr.Row(): gr.HTML('