src/chain_data.py

import os
import traceback
from datetime import datetime, timedelta
from enum import Enum
from threading import Lock
from typing import TypeAlias

import requests
from cachetools import TTLCache, cached
from fiber import constants
from fiber.chain.commitments import _deserialize_commitment_field
from fiber.chain.interface import get_substrate
from fiber.chain.metagraph import Metagraph
from pydantic import BaseModel
from substrateinterface import SubstrateInterface
from substrateinterface.storage import StorageKey

from network_commitments import Decoder
from src import Key, Uid, TIMEZONE

TAOSTATS_API_KEY = os.getenv("TAOSTATS_API_KEY")

Weight: TypeAlias = float
Incentive: TypeAlias = float


class ContestId(Enum):
    FLUX_NVIDIA_4090 = 0
    SDXL_NEWDREAM_NVIDIA_4090 = 1


class Neuron(BaseModel):
    hotkey: str
    coldkey: str
    validator_trust: float
    validator_permit: bool
    incentive: float
    updated: int
    uid: int
    block_number: int


class Commitment(BaseModel):
    url: str
    revision: str
    contest: ContestId
    block: int

    @classmethod
    def decode(cls, decoder: Decoder, block: int):
        provider = decoder.read_str()
        repository = decoder.read_str()
        revision = decoder.read_sized_str(7)
        contest_id = ContestId(decoder.read_uint16())

        return cls(
            url=f"https://{provider}/{repository}",
            revision=revision,
            contest=contest_id,
            block=block
        )


SPEC_VERSION = 8
NET_UID = 39
WEIGHTS_BY_MINER: dict[Key, list[tuple[Key, Weight]]] = {}
VALIDATOR_IDENTITIES: dict[Key, str] = {}

HOTKEYS_BY_UID: dict[Uid, Key] = {}

substrate = get_substrate(subtensor_address=constants.FINNEY_SUBTENSOR_ADDRESS)


def query_subtensor(storage_keys: list[StorageKey], block: int) -> list:
    global substrate
    try:
        return substrate.query_multi(
            storage_keys=storage_keys,
            block_hash=substrate.get_block_hash(block),
        )
    except Exception:
        substrate = get_substrate(subtensor_address=substrate.url)
        raise


def fetch_weights(block: int):
    storage_keys: list[StorageKey] = []
    for hotkey, neuron in get_neurons().items():
        if not neuron.validator_permit: continue
        storage_keys.append(substrate.create_storage_key(
            "SubtensorModule",
            "Weights",
            [NET_UID, neuron.uid]
        ))

    weights = query_subtensor(storage_keys, block)
    WEIGHTS_BY_MINER.clear()

    for hotkey, neuron in get_neurons().items():
        for storage, validator_weights in weights:
            validator_hotkey = HOTKEYS_BY_UID[storage.params[1]]
            if hotkey not in WEIGHTS_BY_MINER:
                WEIGHTS_BY_MINER[hotkey] = []
            weight = 0.0
            for miner_weight in validator_weights:
                if miner_weight[0].value == neuron.uid:
                    weight = miner_weight[1].value / 2 ** 16
                    break
            WEIGHTS_BY_MINER[hotkey].append((validator_hotkey, weight))


def fetch_identities(block: int):
    VALIDATOR_IDENTITIES.clear()
    storage_keys: list[StorageKey] = []
    for hotkey, neuron in get_neurons().items():
        if not neuron.validator_permit: continue
        storage_keys.append(substrate.create_storage_key(
            "SubtensorModule",
            "Identities",
            [neuron.coldkey]
        ))

    identities = query_subtensor(storage_keys, block)
    for hotkey, neuron in get_neurons().items():
        for storage, info in identities:
            if neuron.coldkey != storage.params[0]: continue
            if info != None:  # noqa
                VALIDATOR_IDENTITIES[hotkey] = info.value["name"]
                break


lock = Lock()
@cached(cache=TTLCache(maxsize=1, ttl=12 * 60 * 60), lock=lock)
def fetch_commitments() -> dict[Key, Commitment]:
    substrate = get_substrate(subtensor_address=constants.FINNEY_SUBTENSOR_ADDRESS)
    block = substrate.get_block_number(None) # type: ignore
    metagraph = Metagraph(substrate=substrate, netuid=NET_UID, load_old_nodes=False)
    metagraph.sync_nodes()
    nodes = {
        key: node for key, node
        in metagraph.nodes.items()
    }

    print("Fetching commitments...")
    commitments: dict[Key, Commitment] = {}
    storage_keys: list[StorageKey] = []
    for hotkey, node in nodes.items():
        if node.vtrust > 0:
            continue
        storage_keys.append(substrate.create_storage_key(
            "Commitments",
            "CommitmentOf",
            [NET_UID, hotkey]
        ))

    results = substrate.query_multi(
        storage_keys=storage_keys,
        block_hash=substrate.get_block_hash(block),
    )

    for storage, commitment in results:
        try:
            if not commitment or not commitment.value:
                continue

            fields = commitment.value["info"]["fields"]
            if not fields:
                continue

            field = _deserialize_commitment_field(fields[0])
            if field is None:
                continue

            decoder = Decoder(field[1])
            spec_version = decoder.read_uint16()
            if spec_version != SPEC_VERSION:
                continue

            commitments[storage.params[1]] = Commitment.decode(decoder, int(commitment.value["block"]))
        except:
            traceback.print_exc()

    return commitments


last_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
last_identity_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)


@cached(cache=TTLCache(maxsize=1, ttl=120))
def get_neurons() -> dict[Key, Neuron]:
    response = requests.get(
        "https://api.taostats.io/api/metagraph/latest/v1",
        headers={
            "accept": "application/json",
            "Authorization": TAOSTATS_API_KEY,
        },
        params={
            "netuid": 39,
        }
    )

    response.raise_for_status()

    return {
        neuron["hotkey"]["ss58"]: Neuron(
            hotkey=neuron["hotkey"]["ss58"],
            coldkey=neuron["coldkey"]["ss58"],
            validator_trust=float(neuron["validator_trust"]),
            validator_permit=bool(neuron["validator_permit"]),
            incentive=float(neuron["incentive"]),
            updated=int(neuron["updated"]),
            uid=int(neuron["uid"]),
            block_number=int(neuron["block_number"]),
        ) for neuron in response.json()["data"]
    }


def get_latest_block():
    return max([neuron.block_number for neuron in get_neurons().values()])


def sync_chain():
    global substrate
    global last_sync
    now = datetime.now(TIMEZONE)
    if now - last_sync < timedelta(minutes=5):
        return
    last_sync = now

    try:
        print("Syncing chain...")

        block = get_latest_block()

        for hotkey, neuron in get_neurons().items():
            HOTKEYS_BY_UID[neuron.uid] = hotkey

        fetch_weights(block)

        global last_identity_sync
        if now - last_identity_sync > timedelta(days=1):
            print("Syncing identities...")
            last_identity_sync = now
            fetch_identities(block)

    except Exception:
        print(f"Error occurred while syncing chain")
        traceback.print_exc()
        substrate = SubstrateInterface(substrate.url)