Update gradio, fix not showing all submissions

README.md

@@ -4,7 +4,7 @@ emoji: 🏆
 colorFrom: purple
 colorTo: gray
 sdk: gradio
-sdk_version: 5.11.0
+sdk_version: 5.13.1
 app_file: src/app.py
 pinned: true
 ---

requirements.txt

@@ -1,9 +1,9 @@
 fiber @ git+https://github.com/rayonlabs/[email protected]#egg=fiber[chain]
-gradio==5.11.0
+gradio==5.13.1
 wandb==0.19.2
 substrate-interface==1.7.10
 plotly==5.24.1
 pandas==2.2.3
 packaging==24.2
 netaddr==1.3.0
-cachetools==5.5.0
+cachetools==5.5.1

src/chain_data.py

@@ -1,8 +1,8 @@
 import os
 import traceback
-from dataclasses import dataclass
 from datetime import datetime, timedelta
 from enum import Enum
+from threading import Lock
 from typing import TypeAlias
 
 import requests
@@ -10,6 +10,7 @@ from cachetools import TTLCache, cached
 from fiber import constants
 from fiber.chain.commitments import _deserialize_commitment_field
 from fiber.chain.interface import get_substrate
+from fiber.chain.metagraph import Metagraph
 from pydantic import BaseModel
 from substrateinterface import SubstrateInterface
 from substrateinterface.storage import StorageKey
@@ -18,7 +19,6 @@ from network_commitments import Decoder
 from src import Key, Uid, TIMEZONE
 
 TAOSTATS_API_KEY = os.getenv("TAOSTATS_API_KEY")
-DISABLE_COMMITMENTS_FETCH = int(os.getenv("DISABLE_COMMITMENTS_FETCH") or 0) > 0
 
 Weight: TypeAlias = float
 Incentive: TypeAlias = float
@@ -40,10 +40,8 @@ class Neuron(BaseModel):
     block_number: int
 
 
-@dataclass
-class Commitment:
-    provider: str
-    repository: str
+class Commitment(BaseModel):
+    url: str
     revision: str
     contest: ContestId
     block: int
@@ -56,22 +54,17 @@ class Commitment:
         contest_id = ContestId(decoder.read_uint16())
 
         return cls(
-            provider=provider,
-            repository=repository,
+            url=f"https://{provider}/{repository}",
             revision=revision,
             contest=contest_id,
             block=block
         )
 
-    def get_repo_link(self):
-        return f"https://{self.provider}/{self.repository}"
-
 
 SPEC_VERSION = 8
 NET_UID = 39
 WEIGHTS_BY_MINER: dict[Key, list[tuple[Key, Weight]]] = {}
 VALIDATOR_IDENTITIES: dict[Key, str] = {}
-COMMITMENTS: dict[Key, Commitment] = {}
 
 HOTKEYS_BY_UID: dict[Uid, Key] = {}
 
@@ -136,22 +129,36 @@ def fetch_identities(block: int):
                 break
 
 
-def fetch_commitments(block: int):
-    if DISABLE_COMMITMENTS_FETCH:
-        return
+lock = Lock()
+@cached(cache=TTLCache(maxsize=1, ttl=12 * 60 * 60), lock=lock)
+def fetch_commitments() -> dict[Key, Commitment]:
+    substrate = get_substrate(subtensor_address=constants.FINNEY_SUBTENSOR_ADDRESS)
+    block = substrate.get_block_number(None) # type: ignore
+    metagraph = Metagraph(substrate=substrate, netuid=NET_UID, load_old_nodes=False)
+    metagraph.sync_nodes()
+    nodes = {
+        key: node for key, node
+        in metagraph.nodes.items()
+    }
 
-    COMMITMENTS.clear()
+    print("Fetching commitments...")
+    commitments: dict[Key, Commitment] = {}
     storage_keys: list[StorageKey] = []
-    for hotkey, neuron in get_neurons().items():
-        if neuron.validator_permit: continue
+    for hotkey, node in nodes.items():
+        if node.vtrust > 0:
+            continue
         storage_keys.append(substrate.create_storage_key(
             "Commitments",
             "CommitmentOf",
             [NET_UID, hotkey]
         ))
 
-    commitments = query_subtensor(storage_keys, block)
-    for storage, commitment in commitments:
+    results = substrate.query_multi(
+        storage_keys=storage_keys,
+        block_hash=substrate.get_block_hash(block),
+    )
+
+    for storage, commitment in results:
         try:
             if not commitment or not commitment.value:
                 continue
@@ -169,14 +176,15 @@ def fetch_commitments(block: int):
             if spec_version != SPEC_VERSION:
                 continue
 
-            COMMITMENTS[storage.params[1]] = Commitment.decode(decoder, int(commitment.value["block"]))
+            commitments[storage.params[1]] = Commitment.decode(decoder, int(commitment.value["block"]))
         except:
-            continue
+            traceback.print_exc()
+
+    return commitments
 
 
 last_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
 last_identity_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
-last_commitment_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
 
 
 @cached(cache=TTLCache(maxsize=1, ttl=120))
@@ -207,6 +215,7 @@ def get_neurons() -> dict[Key, Neuron]:
         ) for neuron in response.json()["data"]
     }
 
+
 def get_latest_block():
     return max([neuron.block_number for neuron in get_neurons().values()])
 
@@ -235,11 +244,6 @@ def sync_chain():
             last_identity_sync = now
             fetch_identities(block)
 
-        global last_commitment_sync
-        if now - last_commitment_sync > timedelta(hours=12):
-            print("Syncing commitments...")
-            last_commitment_sync = now
-            fetch_commitments(block)
     except Exception:
         print(f"Error occurred while syncing chain")
         traceback.print_exc()

src/submissions.py

@@ -3,7 +3,7 @@ from enum import Enum
 import gradio as gr
 import pandas as pd
 
-from chain_data import sync_chain, COMMITMENTS
+from chain_data import sync_chain, fetch_commitments
 from src import Key
 from src.chain_data import get_neurons
 from wandb_data import get_current_runs, Run, get_blacklisted_keys
@@ -48,7 +48,7 @@ def create_submissions(submission_filters: list[str]) -> gr.Dataframe:
     sync_chain()
     runs = sorted(get_current_runs(), key=lambda run: run.uid)
 
-    for hotkey, commitment in COMMITMENTS.items():
+    for hotkey, commitment in fetch_commitments().items():
         neuron = get_neurons().get(hotkey)
         if not neuron:
             continue
@@ -56,9 +56,9 @@ def create_submissions(submission_filters: list[str]) -> gr.Dataframe:
 
         row = [
             neuron.uid,
-            f"[{'/'.join(commitment.get_repo_link().split('/')[-2:])}]({commitment.get_repo_link()})",
+            f"[{'/'.join(commitment.url.split('/')[-2:])}]({commitment.url})",
             f"[{commitment.block}](https://taostats.io/block/{commitment.block}/extrinsics)",
-            f"[{commitment.revision}]({commitment.get_repo_link()}/commit/{commitment.revision})",
+            f"[{commitment.revision}]({commitment.url}/commit/{commitment.revision})",
             f"[{hotkey[:6]}...](https://taostats.io/hotkey/{hotkey})",
             f"[{coldkey[:6]}...](https://taostats.io/coldkey/{coldkey})",
             commitment.contest.name,

src/wandb_data.py

@@ -7,7 +7,7 @@ import requests
 import wandb
 from cachetools import TTLCache, cached
 import wandb.apis.public as wapi
-from pydantic import BaseModel, RootModel
+from pydantic import BaseModel
 from substrateinterface import Keypair
 
 from chain_data import VALIDATOR_IDENTITIES, sync_chain
@@ -16,7 +16,7 @@ from src.chain_data import get_neurons
 
 WANDB_RUN_PATH = os.environ["WANDB_RUN_PATH"]
 
-START_DATE = datetime(2024, 11, 29)
+START_DATE = datetime(2025, 1, 28)
 OFFSET_DAYS = 0
 
 BLACKLIST_ENDPOINT = "https://edge-inputs.api.wombo.ai/blacklist"