Spaces:
Runtime error
Runtime error
File size: 2,500 Bytes
ab5f5f1 a3d5a06 ab5f5f1 a3d5a06 ab5f5f1 7c5c6f0 ab5f5f1 73a2adb ab5f5f1 cd9a950 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 |
import gradio as gr
from src.utils import model_hyperlink, process_score
LEADERBOARD_COLUMN_TO_DATATYPE = {
# open llm
"Model 🤗" :"str",
"Arch 🏛️" :"str",
"Params (B)": "number",
"Open LLM Score (%)": "number",
# deployment settings
"DType 📥" :"str",
"Backend 🏭" :"str",
"Optimization 🛠️" :"str",
"Quantization 🗜️" :"str",
# primary measurements
"Prefill Latency (s)": "number",
"Decode Throughput (tokens/s)": "number",
"Allocated Memory (MB)": "number",
"Energy (tokens/kWh)": "number",
# additional measurements
"E2E Latency (s)": "number",
"E2E Throughput (tokens/s)": "number",
"Reserved Memory (MB)": "number",
"Used Memory (MB)": "number",
}
from dataclasses import dataclass
@dataclass
class LeaderboardColumn:
name: str
type: str
LEADERBOARD_COLUMNS = [
LeaderboardColumn("Model 🤗", "str"),
LeaderboardColumn("Arch 🏛️", "str"),
LeaderboardColumn("Params (B)", "number"),
LeaderboardColumn("Open LLM Score (%)", "number"),
LeaderboardColumn("DType 📥", "str"),
LeaderboardColumn("Backend 🏭", "str"),
LeaderboardColumn("Optimization 🛠️", "str"),
LeaderboardColumn("Quantization 🗜️", "str"),
LeaderboardColumn("Prefill Latency (s)", "number"),
LeaderboardColumn("Decode Throughput (tokens/s)", "number"),
LeaderboardColumn("Allocated Memory (MB)", "number"),
LeaderboardColumn("Energy (tokens/kWh)", "number"),
LeaderboardColumn("E2E Latency (s)", "number"),
LeaderboardColumn("E2E Throughput (tokens/s)", "number"),
LeaderboardColumn("Reserved Memory (MB)", "number"),
LeaderboardColumn( "Used Memory (MB)", "number")
]
def process_model(model_name):
link = f"https://huggingface.co/{model_name}"
return model_hyperlink(link, model_name)
def get_leaderboard_df(llm_perf_df):
df = llm_perf_df.copy()
# transform for leaderboard
df["Model 🤗"] = df["Model 🤗"].apply(process_model)
# process quantization for leaderboard
df["Open LLM Score (%)"] = df.apply(
lambda x: process_score(x["Open LLM Score (%)"], x["Quantization 🗜️"]),
axis=1,
)
return df
COLS = [col.name for col in LEADERBOARD_COLUMNS]
TYPES = [col.type for col in LEADERBOARD_COLUMNS]
def create_leaderboard_table(llm_perf_df):
# get dataframe
leaderboard_df = get_leaderboard_df(llm_perf_df)
print(leaderboard_df)
return leaderboard_df
|