Initial commit

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/

README.md

@@ -1,12 +1,12 @@
 ---
 title: Edge Maxxing Dashboard
-emoji: 💻
+emoji: 🏆
 colorFrom: purple
 colorTo: gray
 sdk: gradio
 sdk_version: 5.5.0
 app_file: app.py
-pinned: false
+pinned: true
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,7 @@
+gradio==5.5.0
+bittensor==8.2.0
+wandb==0.18.5
+substrate-interface==1.7.10
+plotly==5.24.1
+pandas==2.2.3
+packaging==24.1

src/app.py

@@ -0,0 +1,40 @@
+import gradio as gr
+
+from chain_data import sync_metagraph
+from leaderboard import create_leaderboard, create_dropdown
+from validator_states import create_validator_states
+from validator_weights import create_weights
+from wandb_data import sync
+
+
+def main():
+    sync_metagraph()
+    sync()
+    with gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}", fill_height=True, fill_width=True) as app:
+        with gr.Tab("Leaderboard") as leaderboard_tab:
+            dropdown = gr.Dropdown()
+            dropdown.attach_load_event(lambda: create_dropdown(), None)
+
+            leaderboard_dataframe = gr.Dataframe()
+            leaderboard_dataframe.attach_load_event(lambda uid: create_leaderboard(uid), None, inputs=[dropdown])
+            leaderboard_tab.select(lambda uid: create_leaderboard(uid), inputs=[dropdown], outputs=[leaderboard_dataframe])
+
+            dropdown.change(lambda uid: create_leaderboard(uid), inputs=[dropdown], outputs=[leaderboard_dataframe])
+
+        with gr.Tab("Validator States") as validator_states_tab:
+            validator_states_dataframe = gr.Dataframe()
+            validator_states_dataframe.attach_load_event(lambda: create_validator_states(), None)
+            validator_states_tab.select(lambda: create_validator_states(), [], [validator_states_dataframe])
+
+        with gr.Tab("Validator Weights") as validator_weights_tab:
+            validator_weights_dataframe = gr.Dataframe()
+            validator_weights_dataframe.attach_load_event(lambda: create_weights(), None)
+            validator_weights_tab.select(lambda: create_weights(), [], [validator_weights_dataframe])
+
+        with gr.Tab("Model Demo"):
+            gr.Label("Coming soon!", show_label=False)
+    app.launch()
+
+
+if __name__ == "__main__":
+    main()

src/chain_data.py

@@ -0,0 +1,91 @@
+from datetime import datetime, timedelta
+from typing import TypeAlias
+
+import bittensor as bt
+
+from wandb_data import Uid, TIMEZONE
+
+Weight: TypeAlias = float
+Incentive: TypeAlias = float
+
+NET_UID = 39
+WEIGHTS_BY_MINER: list[list[tuple[Uid, Weight]]] = []
+INCENTIVES: dict[Uid, Incentive] = {}
+VALIDATOR_IDENTITIES: dict[Uid, str] = {}
+VTRUST: dict[Uid, float] = {}
+UPDATED: dict[Uid, int] = {}
+
+subtensor = bt.subtensor()
+metagraph = bt.metagraph(netuid=NET_UID)
+
+
+def fetch_weights():
+    WEIGHTS_BY_MINER.clear()
+    weights = subtensor.weights(netuid=NET_UID)
+
+    for miner_uid in range(metagraph.n.item()):
+        WEIGHTS_BY_MINER.append([])
+        for validator_uid, validator_weights in weights:
+            if not metagraph.validator_permit[validator_uid]:
+                continue
+
+            weight = 0.0
+            for miner_weight in validator_weights:
+                if miner_weight[0] == miner_uid:
+                    weight = miner_weight[1] / 2 ** 16
+                    break
+            WEIGHTS_BY_MINER[miner_uid].append((validator_uid, weight))
+
+
+def fetch_incentives():
+    INCENTIVES.clear()
+    for i in range(len(metagraph.incentive)):
+        INCENTIVES[i] = metagraph.incentive[i]
+
+def fetch_vtrust():
+    VTRUST.clear()
+    for i in range(len(metagraph.validator_trust)):
+        VTRUST[i] = metagraph.validator_trust[i]
+
+def fetch_updated():
+    UPDATED.clear()
+    block = subtensor.get_current_block()
+    for i in range(len(metagraph.last_update)):
+        UPDATED[i] = block - metagraph.last_update[i]
+
+
+def fetch_identities():
+    VALIDATOR_IDENTITIES.clear()
+    for uid in range(metagraph.n.item()):
+        if not metagraph.validator_permit[uid]:
+            continue
+        identity = subtensor.substrate.query('SubtensorModule', 'Identities', [metagraph.coldkeys[uid]])
+        if identity != None:
+            VALIDATOR_IDENTITIES[uid] = identity.value["name"]
+
+
+last_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
+last_identity_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
+
+
+def sync_metagraph():
+    global last_sync
+    now = datetime.now(TIMEZONE)
+    if now - last_sync < timedelta(minutes=5):
+        return
+
+    print("Syncing metagraph...")
+    last_sync = now
+    metagraph.sync(subtensor=subtensor)
+
+    fetch_weights()
+    fetch_incentives()
+    fetch_vtrust()
+    fetch_updated()
+
+    global last_identity_sync
+    if now - last_identity_sync < timedelta(hours=12):
+        return
+
+    last_identity_sync = now
+    fetch_identities()

src/leaderboard.py

@@ -0,0 +1,58 @@
+import os
+
+import gradio as gr
+import pandas as pd
+
+from wandb_data import get_current_runs
+
+DEFAULT_VALIDATOR_UID = int(os.environ["DEFAULT_VALIDATOR_UID"])
+
+def create_dropdown() -> gr.Dropdown:
+    choices: list[tuple[str, int]] = []
+    runs = get_current_runs()
+    for run in runs:
+        pretty_name = f"{run.uid} - {run.name} ({run.status.name()})"
+        choices.append((pretty_name, run.uid))
+
+    choices = sorted(choices, key=lambda x: x[1])
+
+    default = DEFAULT_VALIDATOR_UID
+    if not default in [uid for _, uid in choices]:
+        default = choices[0][1]
+    return gr.Dropdown(
+        choices,
+        value=default,
+        interactive=True,
+        label="Source Validator"
+    )
+
+def create_leaderboard(validator_uid) -> gr.Dataframe:
+    data: list[list] = []
+    runs = get_current_runs()
+    for run in runs:
+        if run.uid != validator_uid:
+            continue
+        for submission in run.submissions.values():
+            data.append([
+                submission.info.uid,
+                f"[{'/'.join(submission.info.repository.split('/')[-2:])}]({submission.info.repository})",
+                submission.tier,
+                round(submission.score, 3),
+                f"{submission.metrics.generation_time:.3f}s",
+                f"{submission.average_similarity * 100:.3f}%",
+                f"{submission.metrics.size / 1_000_000_000:.3f}GB",
+                f"{submission.metrics.vram_used / 1_000_000_000:.3f}GB",
+                f"{submission.metrics.watts_used:.3f}W",
+                f"{submission.metrics.load_time:.3f}s",
+                f"[{submission.info.block}](https://taostats.io/block/{submission.info.block})",
+                f"[{submission.info.revision}]({submission.info.repository}/commit/{submission.info.revision})",
+                f"[{submission.info.hotkey[:6]}...](https://taostats.io/hotkey/{submission.info.hotkey})",
+            ])
+
+    data.sort(key=lambda x: (-x[2], int(x[10].split('[')[1].split(']')[0])))
+
+    return gr.Dataframe(
+        pd.DataFrame(data, columns=["UID", "Model", "Tier", "Score", "Gen Time", "Similarity", "Size", "VRAM Usage", "Power Usage", "Load Time", "Block", "Revision", "Hotkey"]),
+        datatype=["number", "markdown", "number", "number", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown"],
+        interactive=False,
+    )

src/validator_states.py

@@ -0,0 +1,57 @@
+import statistics
+from datetime import timedelta
+
+import gradio as gr
+import pandas as pd
+from packaging import version
+
+from wandb_data import get_current_runs, Run
+from chain_data import VTRUST, UPDATED
+
+AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD = 180  # 3 minutes
+ETA_WARNING_THRESHOLD = 43200  # 12 hours
+UPDATED_WARNING_THRESHOLD = 1000
+VTRUST_WARNING_THRESHOLD = 0.75
+
+
+def get_latest_version(runs: list[Run]) -> str:
+    latest_version = version.parse("0.0.0")
+    for run in runs:
+        current_version = version.parse(run.version)
+        if current_version > latest_version:
+            latest_version = current_version
+    return str(latest_version)
+
+
+def colorize(val, color: str) -> str:
+    return f"<span style='color: {color}'>{val}</span>"
+
+
+def create_validator_states() -> gr.Dataframe:
+    data: list[list] = []
+    runs = sorted(get_current_runs(), key=lambda run: run.uid)
+    winners = [run.winner_uid for run in runs if run.winner_uid]
+    winner_uid_mode = statistics.mode(winners) if winners else None
+    latest_version = get_latest_version(runs)
+    for run in runs:
+        vtrust = VTRUST.get(run.uid, 0)
+        updated = UPDATED.get(run.uid, 0)
+        data.append([
+            run.uid,
+            run.name,
+            colorize(run.version, "springgreen" if run.version == latest_version else "red"),
+            colorize(run.status.name(), run.status.color()),
+            colorize(run.winner_uid, "springgreen" if winner_uid_mode and run.winner_uid == winner_uid_mode else "orange" if run.winner_uid else "gray"),
+            f"{len(run.submissions) + len(run.invalid_submissions)}/{run.total_submissions}",
+            len(run.invalid_submissions),
+            colorize(f"{timedelta(seconds=int(run.average_benchmark_time))}", "orange" if run.average_benchmark_time > AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD else "springgreen" if run.average_benchmark_time > 0 else "gray"),
+            colorize(f"{timedelta(seconds=run.eta)}", "orange" if run.eta > ETA_WARNING_THRESHOLD else "springgreen" if run.eta > 0 else "gray"),
+            colorize(f"{vtrust:.4f}", "springgreen" if vtrust > VTRUST_WARNING_THRESHOLD else "red"),
+            colorize(updated, "springgreen" if updated < UPDATED_WARNING_THRESHOLD else "red"),
+        ])
+
+    return gr.Dataframe(
+        pd.DataFrame(data, columns=["UID", "Name", "Version", "Status", "Winner", "Tested", "Invalid", "Avg. Benchmark Time", "ETA", "VTrust", "Updated"]),
+        datatype=["number", "markdown", "markdown", "markdown", "markdown", "number", "number", "markdown", "markdown", "markdown", "markdown"],
+        interactive=False,
+    )

src/validator_weights.py

@@ -0,0 +1,53 @@
+import gradio as gr
+import pandas as pd
+
+from chain_data import WEIGHTS_BY_MINER, INCENTIVES, sync_metagraph
+
+
+def get_color_by_weight(weight: float) -> str:
+    if weight < 0.001:
+        return "gray"
+    elif weight < 0.3:
+        r = int(255)
+        g = int((weight / 0.3) * 165)
+        return f"rgb({r}, {g}, 0)"
+    elif weight < 0.8:
+        progress = (weight - 0.3) / 0.5
+        r = int(255 - (progress * 255))
+        g = int(165 + (progress * 90))
+        return f"rgb({r}, {g}, 0)"
+    else:
+        progress = (weight - 0.8) / 0.2
+        g = int(255 - ((1 - progress) * 50))
+        return f"rgb(0, {g}, 0)"
+
+def create_weights() -> gr.Dataframe:
+    data: list[list] = []
+    sync_metagraph()
+
+    headers = ["Miner UID", "Incentive"]
+    datatype = ["number", "markdown"]
+
+    validator_uids = set()
+    for validator_weights in WEIGHTS_BY_MINER:
+        for validator_uid, _ in validator_weights:
+            validator_uids.add(validator_uid)
+
+    for validator_uid in sorted(validator_uids):
+        headers.append(str(validator_uid))
+        datatype.append("markdown")
+
+    for miner_uid, validator_weights in enumerate(WEIGHTS_BY_MINER):
+        incentive = INCENTIVES[miner_uid]
+        row = [miner_uid, f"<span style='color: {get_color_by_weight(incentive)}'>{incentive:.{3}f}</span>"]
+        for _, weight in validator_weights:
+            row.append(f"<span style='color: {get_color_by_weight(weight)}'>{weight:.{3}f}</span>")
+        data.append(row)
+
+    data.sort(key=lambda val: float(val[1].split(">")[1].split("<")[0]), reverse=True)
+
+    return gr.Dataframe(
+        pd.DataFrame(data, columns=headers),
+        datatype=datatype,
+        interactive=False,
+    )

src/wandb_data.py

@@ -0,0 +1,288 @@
+import os
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from enum import Enum
+from typing import TypeAlias
+from zoneinfo import ZoneInfo
+
+import wandb
+import wandb.apis.public as wapi
+from substrateinterface import Keypair
+
+WANDB_RUN_PATH = os.environ["WANDB_RUN_PATH"]
+
+TIMEZONE = ZoneInfo("America/Los_Angeles")
+START_DATE = datetime(2024, 11, 7)
+
+Uid: TypeAlias = int
+
+
+class BenchmarkStatus(Enum):
+    NOT_STARTED = ("Not Started", "orange", False)
+    IN_PROGRESS = ("In Progress", "orange", False)
+    FINISHED = ("Finished", "springgreen", False)
+    INITIALISING = ("Initialising", "orange", False)
+    STOPPED = ("Stopped", "red", True)
+    CRASHED = ("Crashed", "red", True)
+    FAILED = ("Failed", "red", True)
+    UNKNOWN = ("Unknown", "red", True)
+
+    def name(self):
+        return self.value[0]
+
+    def color(self):
+        return self.value[1]
+
+    def failed(self):
+        return self.value[2]
+
+
+@dataclass
+class MetricData:
+    generation_time: float
+    vram_used: float
+    watts_used: float
+    load_time: float
+    size: int
+
+
+@dataclass
+class SubmissionInfo:
+    uid: int
+    hotkey: str
+    repository: str
+    revision: str
+    block: int
+
+
+@dataclass
+class Submission:
+    info: SubmissionInfo
+    metrics: MetricData
+    average_similarity: float
+    min_similarity: float
+    tier: int
+    score: float
+
+
+@dataclass
+class InvalidSubmission:
+    info: SubmissionInfo
+    reason: str
+
+
+@dataclass
+class Run:
+    start_date: datetime
+    version: str
+    uid: int
+    name: str
+    hotkey: str
+    status: BenchmarkStatus
+    average_benchmark_time: float
+    eta: int
+    winner_uid: int | None
+    baseline_metrics: MetricData | None
+    total_submissions: int
+    submissions: dict[Uid, Submission]
+    invalid_submissions: dict[Uid, InvalidSubmission]
+
+
+RUNS: dict[str, list[Run]] = {}
+
+
+def _is_valid_run(run: wapi.Run):
+    required_config_keys = ["hotkey", "uid", "contest", "signature"]
+
+    for key in required_config_keys:
+        if key not in run.config:
+            return False
+
+    validator_hotkey = run.config["hotkey"]
+    contest_name = run.config["contest"]
+
+    signing_message = f"{run.name}:{validator_hotkey}:{contest_name}"
+
+    return Keypair(validator_hotkey).verify(signing_message, run.config["signature"])
+
+
+def _date_from_run(run: wapi.Run) -> datetime:
+    return datetime.strptime(run.created_at, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).astimezone(TIMEZONE)
+
+
+def _status_from_run(run: wapi.Run) -> BenchmarkStatus:
+    match run.state:
+        case "finished":
+            return BenchmarkStatus.STOPPED
+        case "crashed":
+            return BenchmarkStatus.CRASHED
+        case "failed":
+            return BenchmarkStatus.FAILED
+        case "running":
+            if "benchmarking_state" in run.summary:
+                return BenchmarkStatus[run.summary["benchmarking_state"]]
+            else:
+                return BenchmarkStatus.INITIALISING
+        case _:
+            return BenchmarkStatus.UNKNOWN
+
+
+def _add_runs(wandb_runs: list[wapi.Run]):
+    for wandb_run in wandb_runs:
+        if not _is_valid_run(wandb_run):
+            continue
+
+        metrics = wandb_run.summary
+
+        submission_info: dict[Uid, SubmissionInfo] = {}
+        submissions: dict[Uid, Submission] = {}
+        invalid_submissions: dict[Uid, InvalidSubmission] = {}
+
+        baseline_metrics: MetricData | None = None
+        if "baseline" in metrics:
+            baseline = metrics["baseline"]
+            baseline_metrics = MetricData(
+                generation_time=float(baseline["generation_time"]),
+                vram_used=float(baseline["vram_used"]),
+                watts_used=float(baseline["watts_used"]),
+                load_time=float(baseline["load_time"]),
+                size=int(baseline["size"]),
+            )
+
+        if "submissions" in metrics:
+            for uid, submission in metrics["submissions"].items():
+                submission_info[uid] = SubmissionInfo(
+                    uid=uid,
+                    hotkey=submission["hotkey"] if "hotkey" in submission else metrics["benchmarks"][uid]["hotkey"] if uid in metrics["benchmarks"] else "unknown",
+                    # hotkey=submission["hotkey"], # TODO use this once validators update
+                    repository=submission["repository"],
+                    revision=submission["revision"],
+                    block=submission["block"],
+                )
+
+        if "benchmarks" in metrics:
+            for uid, benchmark in metrics["benchmarks"].items():
+                model = benchmark["model"]
+                submissions[uid] = Submission(
+                    info=submission_info[uid],
+                    metrics=MetricData(
+                        generation_time=float(model["generation_time"]),
+                        vram_used=float(model["vram_used"]),
+                        watts_used=float(model["watts_used"]),
+                        load_time=float(model["load_time"]),
+                        size=int(model["size"]),
+                    ),
+                    average_similarity=float(benchmark["average_similarity"]),
+                    min_similarity=float(benchmark["min_similarity"]),
+                    tier=int(benchmark["tier"]),
+                    score=float(benchmark["score"]),
+                )
+
+        if "invalid" in metrics:
+            for uid, reason in metrics["invalid"].items():
+                if not uid in submission_info:
+                    continue
+                invalid_submissions[uid] = InvalidSubmission(
+                    info=submission_info[uid],
+                    reason=reason,
+                )
+
+        status = _status_from_run(wandb_run)
+        winners = sorted(
+            submissions.values(),
+            key=lambda submission: (submission.tier, -submission.info.block),
+            reverse=True,
+        )
+        winner_uid = winners[0].info.uid if winners and status == status.FINISHED else None
+
+        from chain_data import VALIDATOR_IDENTITIES
+        uid = int(wandb_run.config["uid"])
+        hotkey = wandb_run.config["hotkey"]
+        date = _date_from_run(wandb_run)
+        id = wandb_run.id
+        average_benchmark_time = float(wandb_run.summary["average_benchmark_time"]) if "average_benchmark_time" in wandb_run.summary else 0
+        run = Run(
+            start_date=date,
+            version=wandb_run.tags[1][8:],
+            uid=uid,
+            name=VALIDATOR_IDENTITIES.get(uid, hotkey),
+            hotkey=hotkey,
+            status=status,
+            average_benchmark_time=average_benchmark_time,
+            eta=int(average_benchmark_time * (len(submission_info) - len(submissions) - len(invalid_submissions))) if average_benchmark_time else 0,
+            winner_uid=winner_uid,
+            baseline_metrics=baseline_metrics,
+            total_submissions=len(submission_info),
+            submissions=submissions,
+            invalid_submissions=invalid_submissions,
+        )
+
+        if id not in RUNS:
+            RUNS[id] = [run]
+        else:
+            present = False
+            for i, existing_run in enumerate(RUNS[id]):
+                if existing_run.uid == run.uid:
+                    RUNS[id][i] = run
+                    present = True
+                    break
+
+            if not present:
+                RUNS[id].append(run)
+
+
+def _fetch_history(wandb_api: wandb.Api):
+    wandb_runs = wandb_api.runs(
+        WANDB_RUN_PATH,
+        filters={"config.type": "validator", "created_at": {'$gt': str(START_DATE)}},
+        order="+created_at",
+    )
+    _add_runs(wandb_runs)
+
+
+def _fetch_current_runs(wandb_api: wandb.Api, now: datetime):
+    contest_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
+    wandb_runs = wandb_api.runs(
+        WANDB_RUN_PATH,
+        filters={"config.type": "validator", "created_at": {'$gt': str(contest_start)}},
+        order="+created_at",
+    )
+    _add_runs(wandb_runs)
+
+
+last_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
+
+
+def sync():
+    global last_sync
+    now = datetime.now(TIMEZONE)
+    if now - last_sync < timedelta(seconds=30):
+        return
+
+    print("Syncing runs...")
+    last_sync = now
+    wandb_api = wandb.Api()
+    if not RUNS:
+        _fetch_history(wandb_api)
+    else:
+        _fetch_current_runs(wandb_api, now)
+
+
+def get_current_runs() -> list[Run]:
+    sync()
+    from chain_data import sync_metagraph
+    sync_metagraph()
+
+    now = datetime.now(TIMEZONE)
+    today = now.replace(hour=0, minute=0, second=0, microsecond=0)
+    if now.hour < 12:
+        today -= timedelta(days=1)
+
+    current_runs: list[Run] = []
+
+    for runs in RUNS.values():
+        for run in runs:
+            if run.start_date >= today:
+                current_runs.append(run)
+
+    return current_runs