feat: add TG and PP scores

src/components/visualizations.py

@@ -9,6 +9,13 @@ from typing import Optional, Dict, List, Set
 import plotly.graph_objects as go
 
 
+def clean_device_id(device_id: str) -> str:
+    """Extract clean device name from normalized ID by removing platform prefix"""
+    if device_id.startswith("iOS/"):
+        return device_id[4:]  # Remove "iOS/"
+    return device_id
+
+
 def create_performance_plot(
     df: pd.DataFrame, metric: str, title: str, hover_data: List[str] = None
 ):
@@ -106,13 +113,13 @@ def filter_dataframe(df: pd.DataFrame, filters: Dict) -> pd.DataFrame:
     return filtered_df
 
 
-def create_model_size_performance_plot(df: pd.DataFrame, device: str, title: str):
+def create_model_size_performance_plot(df: pd.DataFrame, device_id: str, title: str):
     """Create a plot showing model size vs performance metrics for a specific device"""
     if df.empty:
         return None
 
     # Filter for the selected device
-    device_df = df[df["Device"] == device].copy()
+    device_df = df[df["Normalized Device ID"] == device_id].copy()
     if device_df.empty:
         return None
 
@@ -237,14 +244,24 @@ def render_model_size_performance(df: pd.DataFrame, filters: Dict):
         return
 
     # Get the device with highest performance score
-    top_device = size_perf_df.loc[size_perf_df["performance_score"].idxmax()]["Device"]
-    devices = sorted(size_perf_df["Device"].unique())
-    default_index = devices.index(top_device)
+    top_device_id = size_perf_df.loc[size_perf_df["performance_score"].idxmax()][
+        "Normalized Device ID"
+    ]
+    device_ids = sorted(size_perf_df["Normalized Device ID"].unique())
+    default_index = device_ids.index(top_device_id)
+
+    # Create mapping of normalized IDs to display names
+    device_display_names = {
+        device_id: clean_device_id(device_id) for device_id in device_ids
+    }
 
     # Device selector for size vs performance plots
-    selected_device = st.selectbox(
+    selected_device_id = st.selectbox(
         "Select Device",
-        options=devices,
+        options=device_ids,
+        format_func=lambda x: device_display_names[
+            x
+        ],  # Display clean names in dropdown
         help="Select a device to view its performance across different model sizes",
         key="size_perf_device_selector",
         placeholder="Search for a device...",
@@ -254,8 +271,8 @@ def render_model_size_performance(df: pd.DataFrame, filters: Dict):
     # Create and display the model size vs performance plot
     size_perf_fig = create_model_size_performance_plot(
         size_perf_df,
-        selected_device,
-        f"Model Size vs Performance Metrics for {selected_device}",
+        selected_device_id,
+        f"Model Size vs Performance Metrics for {device_display_names[selected_device_id]}",
     )
 
     if size_perf_fig:
@@ -489,12 +506,6 @@ def render_device_rankings(df: pd.DataFrame):
         st.warning("No data available for device rankings.")
         return
 
-    def clean_device_id(device_id: str) -> str:
-        """Extract clean device name from normalized ID by removing platform prefix"""
-        if device_id.startswith("iOS/"):
-            return device_id[4:]  # Remove "iOS/"
-        return device_id
-
     # Create device summary
     device_summary = (
         df.groupby(["Normalized Device ID", "Platform"])
@@ -502,8 +513,8 @@ def render_device_rankings(df: pd.DataFrame):
             {
                 "performance_score": "max",  # Best score achieved
                 "Model Size": ["min", "max"],  # Size range
-                "Token Generation": "max",  # Best token generation speed
-                "Prompt Processing": "max",  # Best prompt processing speed
+                "tg_score": "max",  # Use normalized TG score
+                "pp_score": "max",  # Use normalized PP score
                 "Model ID": lambda x: ", ".join(sorted(set(x))),  # All models tested
                 "quant_factor": lambda x: sorted(set(x)),  # Quantization levels tested
             }
@@ -518,8 +529,8 @@ def render_device_rankings(df: pd.DataFrame):
         "Best Score",
         "Min Model Size",
         "Max Model Size",
-        "Best TG Speed",
-        "Best PP Speed",
+        "TG Score",
+        "PP Score",
         "Tested Models",
         "Tested Quantizations",
     ]
@@ -545,19 +556,20 @@ def render_device_rankings(df: pd.DataFrame):
         # Format the display columns
         display_df = overall_rankings.copy()
         display_df["Best Score"] = display_df["Best Score"].round(2)
-        display_df["Best TG Speed"] = display_df["Best TG Speed"].round(2)
-        display_df["Best PP Speed"] = display_df["Best PP Speed"].round(2)
+        display_df["TG Score"] = display_df["TG Score"].round(2)
+        display_df["PP Score"] = display_df["PP Score"].round(2)
+
         display_df["Model Size Range"] = display_df.apply(
             lambda x: f"{x['Min Model Size']:.1f}B - {x['Max Model Size']:.1f}B", axis=1
         )
 
         # Select and reorder columns for display
         display_cols = [
-            "Device",
+            "Device",  # Use clean device name for display
             "Platform",
             "Best Score",
-            "Best TG Speed",
-            "Best PP Speed",
+            "TG Score",
+            "PP Score",
             "Model Size Range",
         ]
 
@@ -580,14 +592,14 @@ def render_device_rankings(df: pd.DataFrame):
                 "Best Score": st.column_config.NumberColumn(
                     "Score", help="Overall performance score (0-100)", format="%.2f"
                 ),
-                "Best TG Speed": st.column_config.NumberColumn(
-                    "Best TG Speed (t/s)",
-                    help="Best token generation speed",
+                "TG Score": st.column_config.NumberColumn(
+                    "TG Score",
+                    help="Normalized Token Generation score (0-100)",
                     format="%.2f",
                 ),
-                "Best PP Speed": st.column_config.NumberColumn(
-                    "Best PP Speed (t/s)",
-                    help="Best prompt processing speed",
+                "PP Score": st.column_config.NumberColumn(
+                    "PP Score",
+                    help="Normalized Prompt Processing score (0-100)",
                     format="%.2f",
                 ),
             },
@@ -620,8 +632,8 @@ def render_device_rankings(df: pd.DataFrame):
             .agg(
                 {
                     "performance_score": ["max", "mean"],
-                    "Token Generation": "max",
-                    "Prompt Processing": "max",
+                    "tg_score": "max",  # Use normalized scores
+                    "pp_score": "max",  # Use normalized scores
                     "Model ID": lambda x: ", ".join(sorted(set(x))),
                 }
             )
@@ -635,8 +647,8 @@ def render_device_rankings(df: pd.DataFrame):
             "Size Category",
             "Best Score",
             "Avg Score",
-            "Best TG Speed",
-            "Best PP Speed",
+            "TG Score",
+            "PP Score",
             "Models",
         ]
 
@@ -657,16 +669,16 @@ def render_device_rankings(df: pd.DataFrame):
             # Format scores
             cat_data["Best Score"] = cat_data["Best Score"].round(2)
             cat_data["Avg Score"] = cat_data["Avg Score"].round(2)
-            cat_data["Best TG Speed"] = cat_data["Best TG Speed"].round(2)
-            cat_data["Best PP Speed"] = cat_data["Best PP Speed"].round(2)
+            cat_data["TG Score"] = cat_data["TG Score"].round(2)
+            cat_data["PP Score"] = cat_data["PP Score"].round(2)
 
             display_cols = [
-                "Device",
+                "Device",  # Use clean device name for display
                 "Platform",
                 "Best Score",
                 "Avg Score",
-                "Best TG Speed",
-                "Best PP Speed",
+                "TG Score",
+                "PP Score",
             ]
 
             st.dataframe(
@@ -693,14 +705,14 @@ def render_device_rankings(df: pd.DataFrame):
                     "Avg Score": st.column_config.NumberColumn(
                         "Avg Score", help="Average performance score", format="%.2f"
                     ),
-                    "Best TG Speed": st.column_config.NumberColumn(
-                        "Best TG (t/s)",
-                        help="Best token generation speed",
+                    "TG Score": st.column_config.NumberColumn(
+                        "TG Score",
+                        help="Normalized Token Generation score (0-100)",
                         format="%.2f",
                     ),
-                    "Best PP Speed": st.column_config.NumberColumn(
-                        "Best PP (t/s)",
-                        help="Best prompt processing speed",
+                    "PP Score": st.column_config.NumberColumn(
+                        "PP Score",
+                        help="Normalized Prompt Processing score (0-100)",
                         format="%.2f",
                     ),
                 },
@@ -731,8 +743,8 @@ def render_device_rankings(df: pd.DataFrame):
             .agg(
                 {
                     "performance_score": ["max", "mean"],
-                    "Token Generation": "max",
-                    "Prompt Processing": "max",
+                    "tg_score": "max",
+                    "pp_score": "max",
                     "Model ID": lambda x: ", ".join(sorted(set(x))),
                 }
             )
@@ -746,8 +758,8 @@ def render_device_rankings(df: pd.DataFrame):
             "Quant Factor",
             "Best Score",
             "Avg Score",
-            "Best TG Speed",
-            "Best PP Speed",
+            "TG Score",
+            "PP Score",
             "Models",
         ]
 
@@ -771,16 +783,16 @@ def render_device_rankings(df: pd.DataFrame):
             # Format scores
             quant_data["Best Score"] = quant_data["Best Score"].round(2)
             quant_data["Avg Score"] = quant_data["Avg Score"].round(2)
-            quant_data["Best TG Speed"] = quant_data["Best TG Speed"].round(2)
-            quant_data["Best PP Speed"] = quant_data["Best PP Speed"].round(2)
+            quant_data["TG Score"] = quant_data["TG Score"].round(2)
+            quant_data["PP Score"] = quant_data["PP Score"].round(2)
 
             display_cols = [
                 "Device",
                 "Platform",
                 "Best Score",
                 "Avg Score",
-                "Best TG Speed",
-                "Best PP Speed",
+                "TG Score",
+                "PP Score",
             ]
 
             st.dataframe(
@@ -807,14 +819,14 @@ def render_device_rankings(df: pd.DataFrame):
                     "Avg Score": st.column_config.NumberColumn(
                         "Avg Score", help="Average performance score", format="%.2f"
                     ),
-                    "Best TG Speed": st.column_config.NumberColumn(
-                        "Best TG (t/s)",
-                        help="Best token generation speed",
+                    "TG Score": st.column_config.NumberColumn(
+                        "TG Score",
+                        help="Normalized Token Generation score (0-100)",
                         format="%.2f",
                     ),
-                    "Best PP Speed": st.column_config.NumberColumn(
-                        "Best PP (t/s)",
-                        help="Best prompt processing speed",
+                    "PP Score": st.column_config.NumberColumn(
+                        "PP Score",
+                        help="Normalized Prompt Processing score (0-100)",
                         format="%.2f",
                     ),
                 },

src/core/scoring.py

@@ -110,6 +110,8 @@ def calculate_performance_score(df: pd.DataFrame) -> pd.DataFrame:
         )
         # Return original dataframe with zero scores to avoid breaking the app
         df["performance_score"] = 0
+        df["tg_score"] = 0
+        df["pp_score"] = 0
         df["quant_factor"] = df["Model ID"].apply(
             lambda x: get_quantization_tier(x, std)
         )
@@ -124,22 +126,30 @@ def calculate_performance_score(df: pd.DataFrame) -> pd.DataFrame:
         lambda x: get_quantization_tier(x, std)
     )
 
-    # Combined performance score using model size as direct multiplier
-    standard_df["performance_score"] = (
-        (
-            standard_df["normalized_tg"] * std.TG_WEIGHT
-            + standard_df["normalized_pp"] * std.PP_WEIGHT
-        )
+    # Calculate individual TG and PP scores
+    standard_df["tg_score"] = (
+        standard_df["normalized_tg"]
         * standard_df["Model Size"]  # Direct size multiplier
         * standard_df["quant_factor"]  # Apply quantization penalty
     )
 
-    # Normalize final score to 0-100 range
-    max_score = standard_df["performance_score"].max()
-    if max_score > 0:
-        standard_df["performance_score"] = 100 * (
-            standard_df["performance_score"] / max_score
-        )
+    standard_df["pp_score"] = (
+        standard_df["normalized_pp"]
+        * standard_df["Model Size"]  # Direct size multiplier
+        * standard_df["quant_factor"]  # Apply quantization penalty
+    )
+
+    # Combined performance score using weighted TG and PP scores
+    standard_df["performance_score"] = (
+        standard_df["tg_score"] * std.TG_WEIGHT
+        + standard_df["pp_score"] * std.PP_WEIGHT
+    )
+
+    # Normalize all scores to 0-100 range
+    for score_col in ["performance_score", "tg_score", "pp_score"]:
+        max_score = standard_df[score_col].max()
+        if max_score > 0:
+            standard_df[score_col] = 100 * (standard_df[score_col] / max_score)
 
     # Merge scores back into original dataframe
     df = df.merge(
@@ -149,6 +159,8 @@ def calculate_performance_score(df: pd.DataFrame) -> pd.DataFrame:
                 "Platform",
                 "Model ID",
                 "performance_score",
+                "tg_score",
+                "pp_score",
                 "quant_factor",
             ]
         ],
@@ -158,6 +170,8 @@ def calculate_performance_score(df: pd.DataFrame) -> pd.DataFrame:
 
     # Fill missing scores with 0
     df["performance_score"] = df["performance_score"].fillna(0)
+    df["tg_score"] = df["tg_score"].fillna(0)
+    df["pp_score"] = df["pp_score"].fillna(0)
 
     return df