Add submission filters

README.md

@@ -4,7 +4,7 @@ emoji: 🏆
 colorFrom: purple
 colorTo: gray
 sdk: gradio
-sdk_version: 5.9.1
+sdk_version: 5.11.0
 app_file: src/app.py
 pinned: true
 ---

requirements.txt

@@ -1,6 +1,6 @@
-fiber @ git+https://github.com/rayonlabs/[email protected].1#egg=fiber[chain]
-gradio==5.9.1
-wandb==0.19.1
+fiber @ git+https://github.com/rayonlabs/fiber.git@2.1.0#egg=fiber[chain]
+gradio==5.11.0
+wandb==0.19.2
 substrate-interface==1.7.10
 plotly==5.24.1
 pandas==2.2.3

src/app.py

@@ -3,7 +3,7 @@ import gradio as gr
 from chain_data import sync_metagraph
 from leaderboard import create_leaderboard, create_dropdown, create_baseline
 from model_demo import create_demo
-from submissions import create_submissions
+from submissions import create_submissions, DROPDOWN_OPTIONS
 from validator_states import create_validator_states
 from validator_weights import create_weights
 from wandb_data import sync
@@ -41,13 +41,13 @@ def main():
             include_inactive_checkbox.change(lambda include_inactive: create_weights(include_inactive), [include_inactive_checkbox], [validator_weights_dataframe])
 
         with gr.Tab("Submissions") as submissions_tab:
-            include_blacklisted_checkbox = gr.Checkbox(False, label="Include Blacklisted", container=False)
+            filter_dropdown = gr.Dropdown(choices=DROPDOWN_OPTIONS, value=DROPDOWN_OPTIONS, label="Filter", container=False, multiselect=True)
 
             submissions_dataframe = gr.Dataframe()
-            submissions_dataframe.attach_load_event(lambda include_blacklisted: create_submissions(include_blacklisted), None, [include_blacklisted_checkbox])
-            submissions_tab.select(lambda include_blacklisted: create_submissions(include_blacklisted), [include_blacklisted_checkbox], [submissions_dataframe])
+            submissions_dataframe.attach_load_event(lambda include_blacklisted: create_submissions(include_blacklisted), None, [filter_dropdown])
+            submissions_tab.select(lambda submission_filter: create_submissions(submission_filter), [filter_dropdown], [submissions_dataframe])
 
-            include_blacklisted_checkbox.change(lambda include_blacklisted: create_submissions(include_blacklisted), [include_blacklisted_checkbox], [submissions_dataframe])
+            filter_dropdown.change(lambda submission_filter: create_submissions(submission_filter), [filter_dropdown], [submissions_dataframe])
 
         with gr.Tab("Model Demo"):
             create_demo()

src/chain_data.py

@@ -74,7 +74,7 @@ def query_subtensor(storage_keys: list[StorageKey], block: int) -> list:
             block_hash=substrate.get_block_hash(block),
         )
     except Exception:
-        substrate = get_substrate()
+        substrate = get_substrate(subtensor_address=substrate.url)
         raise
 
 
@@ -214,4 +214,4 @@ def sync_metagraph():
             fetch_commitments(block)
     except Exception as e:
         print(f"Error occurred while syncing metagraph: {e}")
-        substrate = get_substrate()
+        substrate = get_substrate(subtensor_address=substrate.url)

src/submissions.py

@@ -1,28 +1,47 @@
+from enum import Enum
+
 import gradio as gr
 import pandas as pd
 
 from chain_data import sync_metagraph, COMMITMENTS, UIDS_BY_HOTKEY, metagraph
 from src import Key
-from wandb_data import get_current_runs, Run, BLACKLISTED_HOTKEYS, BLACKLISTED_COLDKEYS
+from wandb_data import get_current_runs, Run, BLACKLISTED_HOTKEYS, BLACKLISTED_COLDKEYS, DUPLICATE_SUBMISSIONS
+
+
+class SubmissionStatus(Enum):
+    BLACKLISTED = ("Blacklisted", "gray")
+    DUPLICATE = ("Duplicate", "gray")
+    PENDING = ("Pending", "orange")
+    DONE = ("Done", "springgreen")
+    INVALID = ("Invalid", "red")
+
+    @staticmethod
+    def get_status(run: Run, hotkey: Key, coldkey: Key, block: int) -> "SubmissionStatus":
+        if hotkey in BLACKLISTED_HOTKEYS or coldkey in BLACKLISTED_COLDKEYS:
+            return SubmissionStatus.BLACKLISTED
 
+        if any(
+            submission.hotkey == hotkey
+            for submission in DUPLICATE_SUBMISSIONS
+        ):
+            return SubmissionStatus.DUPLICATE
 
-def get_status(run: Run, hotkey: Key, coldkey: Key, block: int) -> tuple[str, str]:
-    if hotkey in BLACKLISTED_HOTKEYS or coldkey in BLACKLISTED_COLDKEYS:
-        return "Blacklisted", "gray"
+        if hotkey in run.submissions and block > run.submissions[hotkey].info.block and hotkey not in run.invalid_submissions:
+            return SubmissionStatus.PENDING
 
-    if hotkey in run.submissions and block > run.submissions[hotkey].info.block and hotkey not in run.invalid_submissions:
-        return "Pending", "orange"
+        if hotkey in run.submissions:
+            return SubmissionStatus.DONE
 
-    if hotkey in run.submissions:
-        return "Done", "springgreen"
+        if hotkey in run.invalid_submissions:
+            return SubmissionStatus.INVALID
 
-    if hotkey in run.invalid_submissions:
-        return "Invalid", "red"
+        return SubmissionStatus.PENDING
 
-    return "Pending", "orange"
 
+DROPDOWN_OPTIONS = [status.value[0] for status in SubmissionStatus]
 
-def create_submissions(include_blacklisted: bool) -> gr.Dataframe:
+
+def create_submissions(submission_filters: list[str]) -> gr.Dataframe:
     data: list[list] = []
     sync_metagraph()
     runs = sorted(get_current_runs(), key=lambda run: run.uid)
@@ -30,9 +49,6 @@ def create_submissions(include_blacklisted: bool) -> gr.Dataframe:
     for hotkey, commitment in COMMITMENTS.items():
         coldkey = metagraph.nodes[hotkey].coldkey
 
-        if not include_blacklisted and (hotkey in BLACKLISTED_HOTKEYS or coldkey in BLACKLISTED_COLDKEYS):
-            continue
-
         row = [
             UIDS_BY_HOTKEY[hotkey],
             f"[{'/'.join(commitment.get_repo_link().split('/')[-2:])}]({commitment.get_repo_link()})",
@@ -43,9 +59,17 @@ def create_submissions(include_blacklisted: bool) -> gr.Dataframe:
             commitment.contest.name,
         ]
 
+        has_data = False
         for run in runs:
-            status, color = get_status(run, hotkey, coldkey, commitment.block)
-            row.append(f"<span style='color: {color}'>{status}</span>")
+            status = SubmissionStatus.get_status(run, hotkey, coldkey, commitment.block)
+            if status.value[0] in submission_filters:
+                row.append(f"<span style='color: {status.value[1]}'>{status.value[0]}</span>")
+                has_data = True
+            else:
+                row.append("")
+
+        if not has_data:
+            continue
 
         data.append(row)
 

src/wandb_data.py

@@ -6,6 +6,7 @@ from enum import Enum
 import requests
 import wandb
 import wandb.apis.public as wapi
+from pydantic import BaseModel, RootModel
 from substrateinterface import Keypair
 
 from chain_data import UIDS_BY_HOTKEY, VALIDATOR_IDENTITIES, sync_metagraph
@@ -17,9 +18,17 @@ START_DATE = datetime(2024, 11, 29)
 OFFSET_DAYS = 0
 
 BLACKLIST_ENDPOINT = "https://edge-inputs.api.wombo.ai/blacklist"
+DUPLICATE_SUBMISSIONS_ENDPOINT = "https://edge-inputs.api.wombo.ai/duplicate_submissions"
 BLACKLISTED_COLDKEYS = set()
 BLACKLISTED_HOTKEYS = set()
 
+class DuplicateSubmission(BaseModel):
+    hotkey: Key
+    copy_of: Key
+    url: str
+
+DUPLICATE_SUBMISSIONS: list[DuplicateSubmission] = []
+
 
 class BenchmarkStatus(Enum):
     NOT_STARTED = ("Not Started", "orange", False)
@@ -286,6 +295,11 @@ def _fetch_blacklisted_keys():
     for coldkey in data["coldkeys"]:
         BLACKLISTED_COLDKEYS.add(coldkey)
 
+    response = requests.get(DUPLICATE_SUBMISSIONS_ENDPOINT)
+    response.raise_for_status()
+    DUPLICATE_SUBMISSIONS.clear()
+    DUPLICATE_SUBMISSIONS.extend(RootModel[list[DuplicateSubmission]].model_validate_json(response.text).root)
+
 
 last_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)