Add timeout

src/app.py

@@ -8,8 +8,8 @@ from wandb_data import sync
 
 
 def main():
-    sync_metagraph()
-    sync()
+    sync_metagraph(timeout=1000)
+    sync(timeout=1000)
     with gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}", fill_height=True, fill_width=True) as app:
         with gr.Tab("Leaderboard") as leaderboard_tab:
             dropdown = gr.Dropdown()

src/chain_data.py

@@ -1,3 +1,4 @@
+from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timedelta
 from typing import TypeAlias
 
@@ -49,9 +50,8 @@ def fetch_vtrust():
         VTRUST[i] = metagraph.validator_trust[i]
 
 
-def fetch_updated():
+def fetch_updated(block: int):
     UPDATED.clear()
-    block = subtensor.get_current_block()
     for i in range(len(metagraph.last_update)):
         UPDATED[i] = block - metagraph.last_update[i]
 
@@ -70,21 +70,22 @@ last_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
 last_identity_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
 
 
-def sync_metagraph():
-    try:
-        global last_sync
-        now = datetime.now(TIMEZONE)
-        if now - last_sync < timedelta(minutes=5):
-            return
+def sync_metagraph(timeout: int = 10):
+    global last_sync
+    now = datetime.now(TIMEZONE)
+    if now - last_sync < timedelta(minutes=5):
+        return
+    last_sync = now
 
+    def sync_task():
         print("Syncing metagraph...")
-        last_sync = now
-        metagraph.sync(subtensor=subtensor)
+        block = subtensor.get_current_block()
+        metagraph.sync(block=block, subtensor=subtensor)
 
         fetch_weights()
         fetch_incentives()
         fetch_vtrust()
-        fetch_updated()
+        fetch_updated(block)
 
         global last_identity_sync
         if now - last_identity_sync < timedelta(hours=12):
@@ -92,5 +93,12 @@ def sync_metagraph():
 
         last_identity_sync = now
         fetch_identities()
-    except:
-        ...
+
+    with ThreadPoolExecutor(max_workers=1) as executor:
+        future = executor.submit(sync_task)
+        try:
+            future.result(timeout=timeout)
+        except TimeoutError:
+            print("Timed out while syncing metagraph")
+        except Exception as e:
+            print(f"Error occurred while syncing metagraph: {e}")

src/wandb_data.py

@@ -1,4 +1,5 @@
 import os
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from enum import Enum
@@ -262,19 +263,29 @@ def _fetch_current_runs(wandb_api: wandb.Api):
 last_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
 
 
-def sync():
+def sync(timeout: int = 10):
     global last_sync
     now = datetime.now(TIMEZONE)
-    if now - last_sync < timedelta(seconds=30):
+    if now - last_sync < timedelta(seconds=60):
         return
-
-    print("Syncing runs...")
     last_sync = now
-    wandb_api = wandb.Api()
-    if not RUNS:
-        _fetch_history(wandb_api)
-    else:
-        _fetch_current_runs(wandb_api)
+
+    def sync_task():
+        print("Syncing runs...")
+        wandb_api = wandb.Api()
+        if not RUNS:
+            _fetch_history(wandb_api)
+        else:
+            _fetch_current_runs(wandb_api)
+
+    with ThreadPoolExecutor(max_workers=1) as executor:
+        future = executor.submit(sync_task)
+        try:
+            future.result(timeout=timeout)
+        except TimeoutError:
+            print("Timed out while syncing runs")
+        except Exception as e:
+            print(f"Error occurred while syncing runs: {e}")
 
 
 def get_current_runs() -> list[Run]: