Spaces:
Sleeping
Sleeping
File size: 2,565 Bytes
bdee176 a8dfc6b bdee176 fd604c4 811f643 bdee176 7d3a98a 811f643 7d3a98a 811f643 6e75d7c 05121a3 94661bc 6e75d7c 05121a3 94661bc 6e75d7c bdee176 c6b858f 6f014a9 9c11640 6f014a9 bd17ee0 6e75d7c b4b3e6a 6b84b3f bd17ee0 bdee176 bd17ee0 811f643 bdee176 fd604c4 bd17ee0 06d3610 a8dfc6b bdee176 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
import pandas as pd
import gradio as gr
import os
def compare_csv_files():
max_num = 10
df1 = pd.read_csv("fish-speech-1.5.csv")
df2 = pd.read_csv("fish-speech-1.4.csv")
merged_df = pd.merge(df1, df2, on="SourceText", suffixes=("_1.5", "_1.4"))
merged_df["WordErrorRate_Diff"] = merged_df["WordErrorRate_1.5"] - merged_df["WordErrorRate_1.4"]
merged_df["CharacterErrorRate_Diff"] = merged_df["CharacterErrorRate_1.5"] - merged_df["CharacterErrorRate_1.4"]
merged_df["WordErrorRate_Comparison"] = merged_df["WordErrorRate_Diff"].apply(
lambda x: "1.4 is the same as 1.5 (Ignored due to large diff)" if abs(x) > max_num else (
f"1.5 is stronger than 1.4 ({x:.8f})" if x < 0 else (
f"1.4 is stronger than 1.5 ({-x:.8f})" if x > 0 else "1.4 is the same as 1.5 (0)"
)
)
)
merged_df["CharacterErrorRate_Comparison"] = merged_df["CharacterErrorRate_Diff"].apply(
lambda x: "1.4 is the same as 1.5 (Ignored due to large diff)" if abs(x) > max_num else (
f"1.5 is stronger than 1.4 ({x:.8f})" if x < 0 else (
f"1.4 is stronger than 1.5 ({-x:.8f})" if x > 0 else "1.4 is the same as 1.5 (0)"
)
)
)
avg_word_diff = merged_df["WordErrorRate_Diff"].loc[merged_df["WordErrorRate_Diff"].abs() <= max_num].mean()
avg_char_diff = merged_df["CharacterErrorRate_Diff"].loc[merged_df["CharacterErrorRate_Diff"].abs() <= 1].mean()
overall_summary = f"""
<h3>Overall Comparison:</h3>
<p>Average WordErrorRate Difference (excluding large diffs): {f'1.5 is stronger ({avg_word_diff:.8f})' if avg_word_diff < 0 else f'1.4 is stronger ({0 - avg_word_diff:.8f})'}</p>
<p>Average CharacterErrorRate Difference (excluding large diffs): {f'1.5 is stronger ({avg_char_diff:.8f})' if avg_char_diff < 0 else f'1.4 is stronger ({0 - avg_char_diff:.8f})'}</p>
"""
result_html = overall_summary + merged_df[[
"SourceText",
"WordErrorRate_1.5", "WordErrorRate_1.4", "WordErrorRate_Comparison",
"CharacterErrorRate_1.5", "CharacterErrorRate_1.4", "CharacterErrorRate_Comparison",
# "WhisperText_1.5", "WhisperText_1.4"
]].to_html(escape=False, index=False)
return result_html
gr.Interface(
fn=compare_csv_files,
inputs=None,
outputs="html",
title="Fish Speech Benchmark",
description="This is a non-official model performance test from Fish Speech / Whisper Base / More data will be added later (not too much)"
).launch()
|