Add ram metric

src/leaderboard.py

@@ -44,6 +44,7 @@ def create_leaderboard(validator_uid) -> gr.Dataframe:
                 f"{submission.average_similarity * 100:.4f}%",
                 f"{submission.metrics.size / 1024 ** 3:.4f}GB",
                 f"{submission.metrics.vram_used / 1024 ** 3:.4f}GB",
+                f"{submission.metrics.ram_used / 1024 ** 3:.4f}GB",
                 f"{submission.metrics.watts_used:.3f}W",
                 f"{submission.metrics.load_time:.3f}s",
                 f"[{submission.info.block}](https://taostats.io/block/{submission.info.block})",
@@ -51,11 +52,11 @@ def create_leaderboard(validator_uid) -> gr.Dataframe:
                 f"[{hotkey[:6]}...](https://taostats.io/hotkey/{hotkey})",
             ])
 
-    data.sort(key=lambda x: (x[2], int(x[10].split('[')[1].split(']')[0])))
+    data.sort(key=lambda x: (x[2], int(x[11].split('[')[1].split(']')[0])))
 
     return gr.Dataframe(
-        pd.DataFrame(data, columns=["UID", "Model", "Rank", "Score", "Gen Time", "Similarity", "Size", "VRAM Usage", "Power Usage", "Load Time", "Block", "Revision", "Hotkey"]),
-        datatype=["number", "markdown", "number", "number", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown"],
+        pd.DataFrame(data, columns=["UID", "Model", "Rank", "Score", "Gen Time", "Similarity", "Size", "VRAM Used", "RAM Used", "Power Used", "Load Time", "Block", "Revision", "Hotkey"]),
+        datatype=["number", "markdown", "number", "number", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown"],
         interactive=False,
         max_height=800,
     )

src/wandb_data.py

@@ -42,12 +42,13 @@ class BenchmarkStatus(Enum):
 
 
 @dataclass
-class MetricData:
+class Metrics:
     generation_time: float
     vram_used: float
     watts_used: float
     load_time: float
     size: int
+    ram_used: float
 
 
 @dataclass
@@ -61,7 +62,7 @@ class SubmissionInfo:
 @dataclass
 class Submission:
     info: SubmissionInfo
-    metrics: MetricData
+    metrics: Metrics
     average_similarity: float
     min_similarity: float
     rank: int
@@ -80,7 +81,7 @@ class Run:
     step: int
     eta: int
     winner_uid: int | None
-    baseline_metrics: MetricData | None
+    baseline_metrics: Metrics | None
     total_submissions: int
     submissions: dict[Key, Submission]
     invalid_submissions: set[Key]
@@ -137,12 +138,13 @@ def _add_runs(wandb_runs: list[wapi.Run]):
         submissions: dict[Key, Submission] = {}
         invalid_submissions: set[Key] = set()
 
-        baseline_metrics: MetricData | None = None
+        baseline_metrics: Metrics | None = None
         if "baseline" in metrics:
             baseline = metrics["baseline"]
-            baseline_metrics = MetricData(
+            baseline_metrics = Metrics(
                 generation_time=float(baseline["generation_time"]),
                 vram_used=float(baseline["vram_used"]),
+                ram_used=float(baseline.get("ram_used", 0)),
                 watts_used=float(baseline["watts_used"]),
                 load_time=float(baseline["load_time"]),
                 size=int(baseline["size"]),
@@ -173,9 +175,10 @@ def _add_runs(wandb_runs: list[wapi.Run]):
                     continue
                 submissions[hotkey] = Submission(
                     info=submission_info[hotkey],
-                    metrics=MetricData(
+                    metrics=Metrics(
                         generation_time=float(benchmark_metrics["generation_time"]),
                         vram_used=float(benchmark_metrics["vram_used"]),
+                        ram_used=float(benchmark_metrics.get("ram_used", 0)),
                         watts_used=float(benchmark_metrics["watts_used"]),
                         load_time=float(benchmark_metrics["load_time"]),
                         size=int(benchmark_metrics["size"]),