finish

.gitignore

@@ -0,0 +1,3 @@
+my-env/
+.env
+__pycache__/

dataset_util.py

@@ -0,0 +1,13 @@
+import pandas as pd
+from datasets import load_dataset
+def load_data(repo="bittensor-dataset/twitter-text-dataset"):
+    dataset = load_dataset(repo)
+    return dataset
+def get_num_rows(dataset):
+    num_rows = dataset['train'].num_rows
+    return num_rows
+if __name__=="__main__":
+    dataset = load_data()
+    num_rows = dataset['train'].num_rows
+    load_data()
+    

indexing_util.py

@@ -0,0 +1,21 @@
+import redis
+import os
+import dotenv
+dotenv.load_dotenv()
+
+hotkey_daily_indexing = redis.Redis(host = os.getenv("REDIS_HOST"), port = os.getenv("REDIS_PORT"), password=os.getenv("REDIS_PASSWORD"), db = 2)
+hotkey_indexing = redis.Redis(host = os.getenv("REDIS_HOST"), port = os.getenv("REDIS_PORT"), password=os.getenv("REDIS_PASSWORD"), db = 3)
+daily_indexing = redis.Redis(host = os.getenv("REDIS_HOST"), port = os.getenv("REDIS_PORT"), password=os.getenv("REDIS_PASSWORD"), db = 4)
+
+def get_all(redis_db):
+    keys = redis_db.keys()
+    # print(keys)
+    result = []
+    for key in keys:
+        value = redis_db.get(key)
+        result.append((key, value))
+    return result
+if __name__ == "__main__":
+    print(get_all(hotkey_daily_indexing))
+    print(get_all(hotkey_indexing))
+    print(get_all(daily_indexing))

leaderboard.py

@@ -0,0 +1,197 @@
+import gradio as gr
+import pandas as pd
+import matplotlib.pyplot as plt
+from dataset_util import load_data, get_num_rows
+import subnet_util
+import datetime
+import typing
+import indexing_util
+from io import BytesIO
+FONT = """<link href="https://fonts.cdnfonts.com/css/intersect-c-brk" rel="stylesheet">"""
+TITLE_FONT = """<link href="https://fonts.cdnfonts.com/css/promova" rel="stylesheet">"""
+TITLE = """ <h1 align = "center" id = "space-title" class = "intersect"> D3 Subnet Leaderboard</h1> """
+DESCRIPTION = """<marquee><h3 align= "center"> The D3 Subnet, standing for Decentralized Distributed Data Scraping subnet, plays a crucial role in the advancement of artificial intelligence by ensuring ample training data for all Bittensor AI networks. </h3></marquee>"""
+IMAGE = """<a href="https://discord.com/channels/799672011265015819/1161764869280903240" target="_blank"><img src="https://cdn.discordapp.com/attachments/1204940599145267200/1227239850332131388/5CB42426-0E73-4D10-A66A-9E256C6A6183.png?ex=6627af2d&is=66153a2d&hm=02b9870618dd8e2e6bf62a0635f3cc8020221a0c7c2568138070f614e63a2068&" alt="D3 Subnet" style="margin: auto; width: 20%; border: 0;" /></a>"""
+
+last_refresh = None
+demo = gr.Blocks(css="""
+                 .intersect {font-family: 'Intersect C BRK', sans-serif; font-size:40px} 
+                 .promova {font-family: 'Promova', sans-serif; font-size:40px}
+                 """)
+
+twitter_text_dataset = load_data("bittensor-dataset/twitter-text-dataset")
+twitter_text_num_rows = get_num_rows(twitter_text_dataset)
+twitter_image_dataset = load_data("bittensor-dataset/twitter-image-dataset")
+twitter_image_num_rows = get_num_rows(twitter_image_dataset)
+
+tao_price = subnet_util.get_tao_price()
+(subtensor, metagraph) = subnet_util.get_subtensor_and_metagraph()
+last_refresh = datetime.datetime.now()
+miners_data = subnet_util.get_subnet_data(subtensor, metagraph)
+
+
+
+daily_indexing_data = indexing_util.get_all(indexing_util.daily_indexing)
+daily_df = pd.DataFrame(daily_indexing_data, columns=['Date', 'Value'])
+daily_df['Date'] = pd.to_datetime(daily_df['Date'].str.decode('utf-8'))
+daily_df['Value'] = daily_df['Value'].astype(int)
+
+hotkey_indexing_data = indexing_util.get_all(indexing_util.hotkey_indexing)
+hotkey_df = pd.DataFrame(hotkey_indexing_data, columns=['Hotkey', 'Value'])
+hotkey_df['Hotkey'] = hotkey_df['Hotkey'].str.decode('utf-8')
+hotkey_df['Value'] = hotkey_df['Value'].astype(int)
+
+hotkey_daily_indexing_data = indexing_util.get_all(indexing_util.hotkey_daily_indexing)
+hotkey_daily_df = pd.DataFrame(hotkey_daily_indexing_data, columns=['Hotkey_Date', 'Value'])
+hotkey_daily_df_= pd.DataFrame()
+hotkey_daily_df_['Hotkey'] = hotkey_daily_df['Hotkey_Date'].str.decode('utf-8').str.split(' ').str[0]
+hotkey_daily_df_['Date'] = hotkey_daily_df['Hotkey_Date'].str.decode('utf-8').str.split(' ').str[1]
+hotkey_daily_df_['Value'] = hotkey_daily_df['Value'].astype(int)
+
+print(hotkey_daily_df_)
+
+
+
+
+
+
+def leaderboard_data(
+    # show_stale: bool,
+    # scores: typing.Dict[int, typing.Dict[str, typing.Optional[float | str]]],
+    # competition_id: str,
+):
+    value = [
+        [
+            c.hotkey[0:8],
+            c.uid,
+            c.url,
+            c.block,
+        ]
+        for c in miners_data
+        # if c.incentive and c.url[0:8] == "https://"
+    ]
+    return value
+
+with demo:
+    gr.HTML(FONT)
+    gr.HTML(TITLE_FONT)
+    gr.HTML(TITLE)
+    gr.HTML(IMAGE)
+    gr.HTML(DESCRIPTION)
+
+    with gr.Tabs():
+        with gr.Accordion("Dataset Stats"):
+            with gr.Row():
+                with gr.Column(scale=1):
+                    gr.HTML(f"<h2 align = 'center'  style = 'font-size: 25px' >Current Size of Text Dataset: <span style = 'font-size: 30px; color: green;'>{twitter_text_num_rows}</span></h2>")
+                with gr.Column(scale=1):
+                    gr.HTML(f"<h2 align = 'center' style = 'font-size: 25px' >Current Size of Image Dataset: <span style = 'font-size: 30px; color: green;'>{twitter_image_num_rows}</span></h2>")
+        with gr.Accordion("Subnet Stats"):
+            gr.HTML(f"""<h2 align = 'center' class="promova" style = 'font-size: 35px;' > Miner Stats</h2>""")
+            
+            with gr.Row():
+                with gr.Column(scale=1):
+                    gr.BarPlot(
+                        daily_df,
+                        x="Date",
+                        y="Value",
+                        title="Daliy scraped data amount",
+                        # color="Date",
+                        tooltip=["Date", "Value"],
+                        y_lim=[0, 1000],
+                        x_title="Date",
+                        y_title="Amount of data scraped",
+                        height=500,
+                        width=500,
+                        scale=5,
+                        color="Value",
+                        color_legend_position="top",
+                        # elem_classes="daily_scraped_data",
+                    )
+                with gr.Column(scale=1):
+                    gr.BarPlot(
+                        hotkey_df,
+                        x="Hotkey",
+                        y="Value",
+                        title="Scraped data amount of each Miner",
+                        # color="Date",
+                        tooltip=["Hotkey", "Value"],
+                        y_lim=[0, 1000],
+                        x_title="Date",
+                        y_title="Amount of data scraped",
+                        height=500,
+                        width=500,
+                        scale=5,
+                        color="Value",
+                        x_label_angle=-30,
+                        color_legend_position="top",
+                        # elem_classes="daily_scraped_data",
+                    )
+            
+            gr.ScatterPlot(
+                hotkey_daily_df_,
+                x="Date",
+                y="Value",
+                title="Daily scraped data amount of each Miner",
+                # color="Date",
+                tooltip=["Hotkey"],
+                y_lim=[0, 1000],
+                x_title="Date",
+                y_title="Amount of data scraped",
+                height=500,
+                width=1000,
+                scale=5,
+                color="Hotkey",
+                x_label_angle=-30,
+                color_legend_position="top",
+                # elem_classes="daily_scraped_data",
+            )
+        with gr.Tab(label="Miners Data"):
+            class_denominator = sum(
+                miners_data[i].incentive #TODO: emssion to incentive
+                for i in range(0, min(10, len(miners_data)))
+                if miners_data[i].incentive
+            )
+            class_values = {
+                f"(uid={miners_data[i].uid}, hotkey={miners_data[i].hotkey[0:8]}) - {miners_data[i].url} · ${round(miners_data[i].emission * tao_price, 2):,} (τ{round(miners_data[i].emission, 2):,})": miners_data[i].incentive / class_denominator
+                for i in range(0, min(10, len(miners_data)))
+                if miners_data[i].incentive
+            }
+            gr.Label(
+                label="Top 10 Miners",
+                value=class_values,
+                num_top_classes=10,
+            )
+        # miner_table = gr.components.Dataframe(
+        #     value=miners_data
+        # )
+        with gr.Accordion("Miner stats"):
+            gr.HTML(
+                f"""<h3>{last_refresh.strftime("refreshed at %H:%M on %Y-%m-%d")}</h3>"""
+            )
+            # with gr.Tabs():
+            #     for entry in miners_data:
+            #         name = f"uid={entry.uid} : commit={entry.commit[0:8]} : url={entry.url}"
+            #         with gr.Tab(name):
+            #             gr.Chatbot()
+            leaderboard_table = gr.components.Dataframe(
+                value=leaderboard_data(),
+                headers = [
+                    "Hotkey",
+                    "UID",
+                    "Url",
+                    "Block",
+                ],
+                datatype=[
+                    "markdown",
+                    "number",
+                    "markdown",
+                    "number",
+
+                ],
+                elem_id="leaderboard_table",
+                interactive=False,
+                visible=True,
+
+            )
+demo.launch()

requirements.txt

@@ -0,0 +1,8 @@
+gradio
+bittensor
+matplotlib
+pandas
+datasets
+tqdm
+redis
+python-dotenv

subnet_util.py

@@ -0,0 +1,132 @@
+import bittensor as bt
+from bittensor.extrinsics.serving import get_metadata
+from utils import functools, run_in_subprocess
+from tqdm import tqdm
+import concurrent.futures
+import datetime
+import typing
+import time
+import requests
+from dataclasses import dataclass
+
+SUBTENSOR = "finney"
+NETUID = 10
+METAGRAPH_RETRIES = 10
+METAGRAPH_DELAY_SECS = 30
+
+def get_subtensor_and_metagraph() -> typing.Tuple[bt.subtensor, bt.metagraph]:
+    for i in range(0, METAGRAPH_RETRIES):
+        try:
+            print("Connecting to subtensor...")
+            subtensor: bt.subtensor = bt.subtensor(SUBTENSOR)
+            print("Pulling metagraph...")
+            metagraph: bt.metagraph = subtensor.metagraph(NETUID, lite=False)
+            return subtensor, metagraph
+        except:
+            if i == METAGRAPH_RETRIES - 1:
+                raise
+            print(
+                f"Error connecting to subtensor or pulling metagraph, retry {i + 1} of {METAGRAPH_RETRIES} in {METAGRAPH_DELAY_SECS} seconds..."
+            )
+            time.sleep(METAGRAPH_DELAY_SECS)
+    raise RuntimeError()
+
+def get_tao_price() -> float:
+    for i in range(0, METAGRAPH_RETRIES):
+        try:
+            return float(requests.get("https://api.mexc.com/api/v3/avgPrice?symbol=TAOUSDT").json()["price"])
+        except:
+            if i == METAGRAPH_RETRIES - 1:
+                raise
+            time.sleep(METAGRAPH_DELAY_SECS)
+        raise RuntimeError()
+
+
+@dataclass
+class MinerData:
+    uid: int
+    hotkey: str
+    block: int
+    url: str
+    incentive: float
+    emission: float
+
+    @classmethod
+    def from_compressed_str(
+        cls,
+        uid: int,
+        hotkey: str,
+        block: int,
+        cs:str,
+        incentive: float,
+        emission: float,
+    ):
+        """Returns an instance of this class from a compressed string representation"""
+        tokens = cs.split(" ")
+        return MinerData(
+            uid=uid,
+            hotkey=hotkey,
+            block=block,
+            url=tokens[0],
+            incentive=incentive,
+            emission=emission,
+        )
+
+def get_subnet_data(
+    subtensor: bt.subtensor, metagraph: bt.metagraph
+):
+
+    # Function to be executed in a thread
+
+    def fetch_data(uid):
+        hotkey = metagraph.hotkeys[uid]
+        try:
+            partial = functools.partial(
+                get_metadata, subtensor, metagraph.netuid, hotkey
+            )
+            metadata = run_in_subprocess(partial, 30)
+        except Exception as e:
+            return None
+
+        if not metadata:
+            return None
+        commitment = metadata["info"]["fields"][0]
+        hex_data = commitment[list(commitment.keys())[0]][2:]
+        chain_str = bytes.fromhex(hex_data).decode()
+        block = metadata["block"]
+        incentive = metagraph.incentive[uid].nan_to_num().item()
+        emission = (
+            metagraph.emission[uid].nan_to_num().item() * 20
+        )  # convert to daily TAO
+
+        try:
+            model_data = MinerData.from_compressed_str(
+                uid, hotkey, block, chain_str, incentive, emission
+            )
+        except Exception as e:
+            print(f"Error parsing model data for uid {uid}: {e}")
+            return None
+        print(model_data)
+        return model_data
+
+    # Use ThreadPoolExecutor to fetch data in parallel
+    results = []
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        # Prepare the list of futures
+        futures = [executor.submit(fetch_data, uid) for uid in metagraph.uids.tolist()]
+        
+        for future in tqdm(
+            concurrent.futures.as_completed(futures),
+            desc="Metadata for hotkeys",
+            total=len(futures),
+        ):
+            result = future.result()
+            if result and result.url[0] != "{":
+                results.append(result)
+
+    return results
+
+if __name__ == "__main__":
+    (subtensor, metagraph) = get_subtensor_and_metagraph()
+    data = get_subnet_data(subtensor, metagraph)
+    print(data)

utils.py

@@ -0,0 +1,150 @@
+import functools
+import multiprocessing
+import os
+import codecs
+import re
+from typing import Any, Optional, Tuple
+import bittensor as bt
+
+
+# def sopt_check(row):
+
+
+
+def assert_registered(wallet: bt.wallet, metagraph: bt.metagraph) -> int:
+    """Asserts the wallet is a registered miner and returns the miner's UID.
+
+    Raises:
+        ValueError: If the wallet is not registered.
+    """
+    if wallet.hotkey.ss58_address not in metagraph.hotkeys:
+        raise ValueError(
+            f"You are not registered. \nUse: \n`btcli s register --netuid {metagraph.netuid}` to register via burn \n or btcli s pow_register --netuid {metagraph.netuid} to register with a proof of work"
+        )
+    uid = metagraph.hotkeys.index(wallet.hotkey.ss58_address)
+    bt.logging.success(
+        f"You are registered with address: {wallet.hotkey.ss58_address} and uid: {uid}"
+    )
+
+    return uid
+
+
+# def validate_hf_repo_id(repo_id: str) -> Tuple[str, str]:
+#     """Verifies a Hugging Face repo id is valid and returns it split into namespace and name.
+
+#     Raises:
+#         ValueError: If the repo id is invalid.
+#     """
+
+#     if not repo_id:
+#         raise ValueError("Hugging Face repo id cannot be empty.")
+
+#     if not 3 < len(repo_id) <= ModelId.MAX_REPO_ID_LENGTH:
+#         raise ValueError(
+#             f"Hugging Face repo id must be between 3 and {ModelId.MAX_REPO_ID_LENGTH} characters. Got={repo_id}"
+#         )
+
+#     parts = repo_id.split("/")
+#     if len(parts) != 2:
+#         raise ValueError(
+#             f"Hugging Face repo id must be in the format <org or user name>/<repo_name>. Got={repo_id}"
+#         )
+
+#     return parts[0], parts[1]
+
+
+# def get_hf_url(model_metadata: ModelMetadata) -> str:
+#     """Returns the URL to the Hugging Face repo for the provided model metadata."""
+#     return f"https://huggingface.co/{model_metadata.id.namespace}/{model_metadata.id.name}/tree/{model_metadata.id.commit}"
+
+
+def _wrapped_func(func: functools.partial, queue: multiprocessing.Queue):
+    try:
+        result = func()
+        queue.put(result)
+    except (Exception, BaseException) as e:
+        # Catch exceptions here to add them to the queue.
+        queue.put(e)
+
+
+def run_in_subprocess(func: functools.partial, ttl: int, mode="fork") -> Any:
+    """Runs the provided function on a subprocess with 'ttl' seconds to complete.
+
+    Args:
+        func (functools.partial): Function to be run.
+        ttl (int): How long to try for in seconds.
+
+    Returns:
+        Any: The value returned by 'func'
+    """
+    ctx = multiprocessing.get_context(mode)
+    queue = ctx.Queue()
+    process = ctx.Process(target=_wrapped_func, args=[func, queue])
+
+    process.start()
+
+    process.join(timeout=ttl)
+
+    if process.is_alive():
+        process.terminate()
+        process.join()
+        raise TimeoutError(f"Failed to {func.func.__name__} after {ttl} seconds")
+
+    # Raises an error if the queue is empty. This is fine. It means our subprocess timed out.
+    result = queue.get(block=False)
+
+    # If we put an exception on the queue then raise instead of returning.
+    if isinstance(result, Exception):
+        raise result
+    if isinstance(result, BaseException):
+        raise Exception(f"BaseException raised in subprocess: {str(result)}")
+
+    return result
+
+
+def get_version() -> str:
+    """
+    Retrieves the version.
+
+    """
+    base_directory = os.path.dirname(os.path.abspath(__file__))
+    with codecs.open(os.path.join(base_directory, '../__init__.py'), encoding='utf-8') as init_file:
+        version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", init_file.read(), re.M)
+        version = version_match.group(1)
+        return version
+
+def upgrade_version():
+    """
+    Upgrade if there is a new version available
+
+    """
+    local_version = get_version()
+    bt.logging.info(f"You are using v{local_version}")
+    try:
+        os.system("git pull origin main > /dev/null 2>&1")
+        remote_version = get_version()
+        if local_version != remote_version:
+            os.system("python3 -m pip install -e . > /dev/null 2>&1")
+            bt.logging.info(f"⏫ Upgraded to v{remote_version}")
+            os._exit(0)
+    except Exception as e:
+        bt.logging.error(f"❌ Error occured while upgrading the version : {e}")
+        
+def save_version(filepath: str, version: int):
+    """Saves a version to the provided filepath."""
+    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+    with open(filepath, "w") as f:
+        f.write(str(version))
+
+
+def move_file_if_exists(src: str, dst: str) -> bool:
+    """Moves a file from src to dst if it exists.
+
+    Returns:
+        bool: True if the file was moved, False otherwise.
+    """
+    if os.path.exists(src) and not os.path.exists(dst):
+        os.makedirs(os.path.dirname(dst), exist_ok=True)
+        os.replace(src, dst)
+        return True
+    return False