Add submissions tab

src/app.py

@@ -2,6 +2,7 @@ import gradio as gr
 
 from chain_data import sync_metagraph
 from leaderboard import create_leaderboard, create_dropdown
+from submissions import create_submissions
 from validator_states import create_validator_states
 from validator_weights import create_weights
 from wandb_data import sync
@@ -35,6 +36,11 @@ def main():
 
             include_inactive_checkbox.change(lambda include_inactive: create_weights(include_inactive), [include_inactive_checkbox], [validator_weights_dataframe])
 
+        with gr.Tab("Submissions") as submissions_tab:
+            submissions_dataframe = gr.Dataframe()
+            submissions_dataframe.attach_load_event(lambda: create_submissions(), None)
+            submissions_tab.select(lambda: create_submissions(), [], [submissions_dataframe])
+
         with gr.Tab("Model Demo"):
             gr.Label("Coming soon!", show_label=False)
     app.launch()

src/chain_data.py

@@ -1,21 +1,58 @@
 from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
 from datetime import datetime, timedelta
+from enum import Enum
 from math import ceil
 from typing import TypeAlias
 
 from fiber.chain.interface import get_substrate
 from fiber.chain.metagraph import Metagraph
 from fiber.chain.models import Node
+from fiber.chain.commitments import _deserialize_commitment_field
 from substrateinterface.storage import StorageKey
 
+from network_commitments import Decoder
 from wandb_data import Hotkey, Uid, TIMEZONE
 
 Weight: TypeAlias = float
 Incentive: TypeAlias = float
 
+class ContestId(Enum):
+    SDXL_APPLE_SILICON = 0
+    SDXL_NEWDREAM_NVIDIA_4090 = 1
+    FLUX_NVIDIA_4090 = 2
+
+@dataclass
+class Commitment:
+    provider: str
+    repository: str
+    revision: str
+    contest: ContestId
+    block: int
+
+    @classmethod
+    def decode(cls, decoder: Decoder, block: int):
+        provider = decoder.read_str()
+        repository = decoder.read_str()
+        revision = decoder.read_sized_str(7)
+        contest_id = ContestId(decoder.read_uint16())
+
+        return cls(
+            provider=provider,
+            repository=repository,
+            revision=revision,
+            contest=contest_id,
+            block=block
+        )
+
+    def get_repo_link(self):
+        return f"https://{self.provider}/{self.repository}"
+
+SPEC_VERSION = 7
 NET_UID = 39
 WEIGHTS_BY_MINER: dict[Hotkey, list[tuple[Hotkey, Weight]]] = {}
 VALIDATOR_IDENTITIES: dict[Hotkey, str] = {}
+COMMITMENTS: dict[Hotkey, Commitment] = {}
 UPDATED: dict[Hotkey, int] = {}
 
 UIDS_BY_HOTKEY: dict[Hotkey, Uid] = {}
@@ -96,9 +133,44 @@ def fetch_identities(block: int):
                 VALIDATOR_IDENTITIES[hotkey] = info.value["name"]
                 break
 
+def fetch_commitments(block: int):
+    COMMITMENTS.clear()
+    storage_keys: list[StorageKey] = []
+    for hotkey, node in metagraph.nodes.items():
+        if is_validator(node): continue
+        storage_keys.append(substrate.create_storage_key(
+            "Commitments",
+            "CommitmentOf",
+            [metagraph.netuid, hotkey]
+        ))
+
+    commitments = query_subtensor(storage_keys, block)
+    for storage, commitment in commitments:
+        try:
+            if not commitment:
+                continue
+
+            fields = commitment.value["info"]["fields"]
+            if not fields:
+                continue
+
+            field = _deserialize_commitment_field(fields[0])
+            if field is None:
+                continue
+
+            decoder = Decoder(field[1])
+            spec_version = decoder.read_uint16()
+            if spec_version != SPEC_VERSION:
+                continue
+
+            COMMITMENTS[storage.params[1]] = Commitment.decode(decoder, int(commitment.value["block"]))
+        except:
+            continue
+
 
 last_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
 last_identity_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
+last_commitment_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
 
 
 def sync_metagraph(timeout: int = 10):
@@ -121,11 +193,16 @@ def sync_metagraph(timeout: int = 10):
         fetch_updated(block)
 
         global last_identity_sync
-        if now - last_identity_sync < timedelta(hours=12):
-            return
-
-        last_identity_sync = now
-        fetch_identities(block)
+        if now - last_identity_sync > timedelta(days=1):
+            print("Syncing identities...")
+            last_identity_sync = now
+            fetch_identities(block)
+
+        global last_commitment_sync
+        if now - last_commitment_sync > timedelta(hours=12):
+            print("Syncing commitments...")
+            last_commitment_sync = now
+            fetch_commitments(block)
 
     with ThreadPoolExecutor(max_workers=1) as executor:
         future = executor.submit(sync_task)

src/leaderboard.py

@@ -55,4 +55,5 @@ def create_leaderboard(validator_uid) -> gr.Dataframe:
         pd.DataFrame(data, columns=["UID", "Model", "Tier", "Score", "Gen Time", "Similarity", "Size", "VRAM Usage", "Power Usage", "Load Time", "Block", "Revision", "Hotkey"]),
         datatype=["number", "markdown", "number", "number", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown"],
         interactive=False,
+        max_height=800,
     )

src/network_commitments.py

@@ -0,0 +1,48 @@
+from struct import pack, unpack
+
+
+_UINT_16_SIZE = 2
+_UINT_32_SIZE = 4
+
+
+class Decoder:
+    _position: int
+    _data: bytes
+
+    def __init__(self, data: bytes):
+        self._position = 0
+        self._data = data
+
+    def read_uint16(self):
+        value = int.from_bytes(self._data[self._position:self._position + _UINT_16_SIZE], "big")
+        self._position += _UINT_16_SIZE
+
+        return value
+
+    def read_uint32(self):
+        value = int.from_bytes(self._data[self._position:self._position + _UINT_32_SIZE], "big")
+        self._position += _UINT_32_SIZE
+
+        return value
+
+    def read_float(self) -> float:
+        return unpack(">f", pack(">L", self.read_uint32()))[0]
+
+    def read_str(self):
+        length = self._data[self._position]
+        self._position += 1
+
+        value = self._data[self._position:self._position + length]
+        self._position += length
+
+        return value.decode()
+
+    def read_sized_str(self, length: int):
+        value = self._data[self._position:self._position + length]
+        self._position += length
+
+        return value.decode()
+
+    @property
+    def eof(self):
+        return self._position >= len(self._data)

src/submissions.py

@@ -0,0 +1,48 @@
+import os
+
+import gradio as gr
+import pandas as pd
+
+from chain_data import sync_metagraph, COMMITMENTS, UIDS_BY_HOTKEY
+from wandb_data import get_current_runs, Run, Uid
+
+DEFAULT_VALIDATOR_UID = int(os.environ["DEFAULT_VALIDATOR_UID"])
+
+def get_status(run: Run, uid: Uid, block: int) -> tuple[str, str]:
+    if all(not submission.get(uid) or block > submission[uid].info.block for submission in [run.submissions, run.invalid_submissions]):
+        return "Pending", "orange"
+
+    if uid in run.submissions:
+        return "Benchmarked", "springgreen"
+    elif uid in run.invalid_submissions:
+        return "Invalid", "red"
+    else:
+        return "Pending", "orange"
+
+def create_submissions() -> gr.Dataframe:
+    data: list[list] = []
+    sync_metagraph()
+    for run in get_current_runs():
+        if not run.uid == DEFAULT_VALIDATOR_UID:
+            continue
+        for hotkey, commitment in COMMITMENTS.items():
+            uid = UIDS_BY_HOTKEY[hotkey]
+            status, color = get_status(run, uid, commitment.block)
+            data.append([
+                uid,
+                f"[{'/'.join(commitment.get_repo_link().split('/')[-2:])}]({commitment.get_repo_link()})",
+                f"[{commitment.block}](https://taostats.io/block/{commitment.block})",
+                f"[{commitment.revision}]({commitment.get_repo_link()}/commit/{commitment.revision})",
+                f"[{hotkey[:6]}...](https://taostats.io/hotkey/{hotkey})",
+                commitment.contest.name,
+                f"<span style='color: {color}'>{status}</span>"
+            ])
+
+    data.sort(key=lambda x: int(x[2].split('[')[1].split(']')[0]), reverse=True)
+
+    return gr.Dataframe(
+        pd.DataFrame(data, columns=["UID", "Model", "Block", "Revision", "Hotkey", "Contest", "Status"]),
+        datatype=["number", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown"],
+        interactive=False,
+        max_height=800,
+    )

src/validator_states.py

@@ -54,4 +54,5 @@ def create_validator_states() -> gr.Dataframe:
         pd.DataFrame(data, columns=["UID", "Name", "Version", "Status", "Winner", "Tested", "Invalid", "Avg. Benchmark Time", "ETA", "VTrust", "Updated"]),
         datatype=["number", "markdown", "markdown", "markdown", "markdown", "number", "number", "markdown", "markdown", "markdown", "markdown"],
         interactive=False,
+        max_height=800,
     )

src/validator_weights.py

@@ -66,4 +66,5 @@ def create_weights(include_inactive: bool) -> gr.Dataframe:
         pd.DataFrame(data, columns=headers),
         datatype=datatype,
         interactive=False,
+        max_height=800,
     )

src/wandb_data.py

@@ -154,7 +154,7 @@ def _add_runs(wandb_runs: list[wapi.Run]):
         if "submissions" in metrics:
             for uid, submission in metrics["submissions"].items():
                 submission_info[uid] = SubmissionInfo(
-                    uid=uid,
+                    uid=int(uid),
                     hotkey=submission["hotkey"],
                     repository=submission["repository"],
                     revision=submission["revision"],
@@ -166,7 +166,7 @@ def _add_runs(wandb_runs: list[wapi.Run]):
                 model = benchmark["model"]
                 if uid not in submission_info:
                     continue
-                submissions[uid] = Submission(
+                submissions[int(uid)] = Submission(
                     info=submission_info[uid],
                     metrics=MetricData(
                         generation_time=float(model["generation_time"]),
@@ -185,7 +185,7 @@ def _add_runs(wandb_runs: list[wapi.Run]):
             for uid, reason in metrics["invalid"].items():
                 if not uid in submission_info:
                     continue
-                invalid_submissions[uid] = InvalidSubmission(
+                invalid_submissions[int(uid)] = InvalidSubmission(
                     info=submission_info[uid],
                     reason=reason,
                 )