guard and retry metagraph sync, filter out models without loss

app.py

@@ -8,6 +8,7 @@ import wandb
 import math
 import os
 import statistics
+import time
 from dotenv import load_dotenv
 from huggingface_hub import HfApi
 from apscheduler.schedulers.background import BackgroundScheduler
@@ -23,9 +24,20 @@ H4_TOKEN = os.environ.get("H4_TOKEN", None)
 API = HfApi(token=H4_TOKEN)
 REPO_ID = "NousResearch/finetuning_subnet_leaderboard"
 MAX_AVG_LOSS_POINTS = 5
+METAGRAPH_RETRIES = 5
+METAGRAPH_DELAY_SECS = 3
 
-subtensor = bt.subtensor("finney")
-metagraph: bt.metagraph = subtensor.metagraph(6, lite=False)
+def get_subtensor_and_metagraph() -> typing.Tuple[bt.subtensor, bt.metagraph]:
+    subtensor: bt.subtensor = bt.subtensor("finney")
+    for i in range(0, METAGRAPH_RETRIES):
+        try:
+            metagraph: bt.metagraph = subtensor.metagraph(6, lite=False)
+            return subtensor, metagraph
+        except:
+            if i == METAGRAPH_RETRIES - 1:
+                raise
+            time.sleep(METAGRAPH_DELAY_SECS)
+    raise RuntimeError()
 
 @dataclass
 class ModelData:
@@ -69,7 +81,7 @@ def print_validator_weights(metagraph: bt.metagraph):
                 if weight > 0:
                     print(f"  {ouid} = {weight}")
 
-def get_subnet_data(metagraph: bt.metagraph) -> typing.List[ModelData]:
+def get_subnet_data(subtensor: bt.subtensor, metagraph: bt.metagraph) -> typing.List[ModelData]:
     result = []
     for uid in metagraph.uids.tolist():
         hotkey = metagraph.hotkeys[uid]
@@ -113,10 +125,11 @@ def get_avg_loss(uids: typing.List[int]) -> typing.Dict[int, float]:
             break
     return result
 
+subtensor, metagraph = get_subtensor_and_metagraph()
 
 tao_price = get_tao_price()
 
-leaderboard_df = get_subnet_data(metagraph)
+leaderboard_df = get_subnet_data(subtensor, metagraph)
 leaderboard_df.sort(key=lambda x: x.incentive, reverse=True)
 
 losses = get_avg_loss([x.uid for x in leaderboard_df])
@@ -137,6 +150,7 @@ with demo:
             c.block
         ] for c in leaderboard_df
     ]
+    value = [x for x in value if x[2] != ""] # filter out anything without a loss
     leaderboard_table = gr.components.Dataframe(
         value=value,
         headers=["Name", "Rewards / Day", "Last Average Loss", "UID", "Block"],