added llm perf score

app.py

@@ -1,4 +1,5 @@
 import os
+import math
 import gradio as gr
 import pandas as pd
 import plotly.express as px
@@ -16,8 +17,8 @@ from src.utils import (
     restart_space,
     load_dataset_repo,
     make_clickable_model,
-    make_clickable_score,
-    num_to_str,
+    # make_clickable_score,
+    # num_to_str,
 )
 from src.assets.css_html_js import custom_css, custom_js
 
@@ -32,12 +33,12 @@ COLUMNS_MAPPING = {
     "backend.torch_dtype": "Load Dtype 📥",
     "optimizations": "Optimizations 🛠️",
     #
+    "perf": "Open LLM-Perf Score ⬆️",
+    #
     "generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️",
+    "score": "Open LLM Score ⬆️",
     "forward.peak_memory(MB)": "Peak Memory (MB) ⬇️",
-    "score": "Average Open LLM Score ⬆️",
-    #
-    "composite_score": "Composite Score ⬆️",
-    "num_params": "#️⃣ Parameters 📏",
+    "num_params": "#️⃣ Parameters (M) 📏",
 }
 COLUMNS_DATATYPES = [
     "markdown",
@@ -47,12 +48,12 @@ COLUMNS_DATATYPES = [
     #
     "number",
     "number",
-    "markdown",
     #
     "number",
-    "str",
+    "number",
+    "number",
 ]
-SORTING_COLUMN = ["Composite Score ⬆️"]
+SORTING_COLUMN = ["Open LLM-Perf Score ⬆️"]
 
 
 llm_perf_dataset_repo = load_dataset_repo(LLM_PERF_DATASET_REPO, OPTIMUM_TOKEN)
@@ -62,36 +63,46 @@ def get_benchmark_df(benchmark="1xA100-80GB"):
     if llm_perf_dataset_repo:
         llm_perf_dataset_repo.git_pull()
 
-    # load
+    # load and merge
     bench_df = pd.read_csv(f"./llm-perf-dataset/reports/{benchmark}.csv")
     scores_df = pd.read_csv(f"./llm-perf-dataset/reports/open-llm-leaderboard.csv")
     bench_df = bench_df.merge(scores_df, on="model", how="left")
+
+    # filter out models with no score
     bench_df = bench_df[bench_df["score"].notna()]
 
+    # create composite score
+    score_distance = 100 - bench_df["score"]
+    latency_distance = bench_df["generate.latency(s)"]
+    bench_df["perf"] = 1 / math.sqrt(score_distance**2 + latency_distance**2)
+    # normalize between 0 and 100
+    bench_df["perf"] = (
+        (bench_df["perf"] - bench_df["perf"].min())
+        / (bench_df["perf"].max() - bench_df["perf"].min())
+        * 100
+    )
+    # round to 2 decimals
+    bench_df["perf"] = bench_df["perf"].round(2)
+
+    # add optimizations
     bench_df["optimizations"] = bench_df[
         ["backend.bettertransformer", "backend.load_in_8bit", "backend.load_in_4bit"]
     ].apply(
         lambda x: ", ".join(
-            sum(
+            filter(
+                lambda x: x != "",
                 [
-                    ["BetterTransformer"] if x[0] == True else [],
-                    ["LLM.int8"] if x[1] == True else [],
-                    ["LLM.fp4"] if x[2] == True else [],
+                    "BetterTransformer" if x[0] == True else "",
+                    "LLM.int8" if x[1] == True else "",
+                    "LLM.fp4" if x[2] == True else "",
                 ],
-                [],
-            )
+            ),
         )
-        if any(x)
+        if any([x[0] == True, x[1] == True, x[2] == True])
         else "None",
         axis=1,
     )
 
-    # create composite score
-    normalized_score = 100 - bench_df["score"]
-    normalized_latency = bench_df["generate.latency(s)"]
-    # normalized_memory = (bench_df["forward.peak_memory(MB)"].max()-bench_df["forward.peak_memory(MB)"])/(bench_df["forward.peak_memory(MB)"].max()-bench_df["forward.peak_memory(MB)"].min())
-    bench_df["composite_score"] = normalized_score + normalized_latency
-
     return bench_df
 
 
@@ -102,13 +113,12 @@ def get_benchmark_table(bench_df):
     bench_df.rename(columns=COLUMNS_MAPPING, inplace=True)
     # sort
     bench_df.sort_values(by=SORTING_COLUMN, ascending=True, inplace=True)
-
     # transform
     bench_df["Model 🤗"] = bench_df["Model 🤗"].apply(make_clickable_model)
-    bench_df["#️⃣ Parameters 📏"] = bench_df["#️⃣ Parameters 📏"].apply(num_to_str)
-    bench_df["Average Open LLM Score ⬆️"] = bench_df["Average Open LLM Score ⬆️"].apply(
-        make_clickable_score
+    bench_df["#️⃣ Parameters (M) 📏"] = bench_df["#️⃣ Parameters 📏"].apply(
+        lambda x: int(x / (1024 * 1024))
     )
+
     return bench_df
 
 
@@ -144,7 +154,7 @@ def get_benchmark_plot(bench_df):
             "yanchor": "top",
         },
         xaxis_title="Per 1000 Tokens Latency (s)",
-        yaxis_title="Average Open LLM Score",
+        yaxis_title="Open LLM Score",
         legend_title="Model Type and Backend",
         width=1200,
         height=600,
@@ -159,8 +169,8 @@ def get_benchmark_plot(bench_df):
                 "Optimizations: %{customdata[3]}",
                 "Peak Memory (MB): %{customdata[4]}",
                 "Throughput (tokens/s): %{customdata[5]}",
-                "Average Open LLM Score: %{y}",
                 "Per 1000 Tokens Latency (s): %{x}",
+                "Open LLM Score: %{y}",
             ]
         )
     )
@@ -232,8 +242,8 @@ with demo:
         with gr.Column(scale=1):
             with gr.Box():
                 score_slider = gr.Slider(
-                    label="Average Open LLM Score 📈",
-                    info="🎚️ Slide to minimum Average Open LLM score",
+                    label="Open LLM Score 📈",
+                    info="🎚️ Slide to minimum Open LLM score",
                     value=0,
                     elem_id="threshold-slider",
                 )