Add new files for pulling data and template for metagraph dashboard

meta_plotting.py

@@ -0,0 +1,48 @@
+import numpy as np
+import plotly.express as px
+
+def plot_trace(df, col='emission', agg='mean', ntop=10, hotkeys=None, hotkey_regex=None, abbrev=8, type='Miners'):
+    
+    if hotkeys is not None:
+        df = df.loc[df.hotkey.isin(hotkeys)]
+    if hotkey_regex is not None:
+        df = df.loc[df.hotkey.str.contains(hotkey_regex)]
+        
+    top_miners = df.groupby('hotkey')[col].agg(agg).sort_values(ascending=False)
+    
+    stats = df.loc[df.hotkey.isin(top_miners.index[:ntop])].sort_values(by=['timestamp'])
+    
+    stats['hotkey_abbrev'] = stats.hotkey.str[:abbrev]
+    stats['coldkey_abbrev'] = stats.coldkey.str[:abbrev]
+    stats['rank'] = stats.hotkey.map({k:i for i,k in enumerate(top_miners.index, start=1)})
+    
+    return px.line(stats.sort_values(by=['timestamp','rank']),
+                    x='timestamp', y=col, color='coldkey_abbrev', line_group='hotkey_abbrev',
+                    hover_data=['hotkey','rank'],
+                    labels={col:col.title(),'timestamp':'','coldkey_abbrev':f'Coldkey (first {abbrev} chars)','hotkey_abbrev':f'Hotkey (first {abbrev} chars)'},
+                    title=f'Top {ntop} {type}, by {col.title()}',
+                    template='plotly_white', width=800, height=600,
+                    ).update_traces(opacity=0.7)
+    
+    
+def plot_cabals(df, sel_col='coldkey', count_col='hotkey', values=None, ntop=10, abbr=8):
+    
+    if values is None:
+        values = df[sel_col].value_counts().sort_values(ascending=False).index[:ntop].tolist()
+        print(f'Automatically selected {sel_col!r} = {values!r}')
+        
+    df = df.loc[df[sel_col].isin(values)]
+    rates = df.groupby(['timestamp',sel_col])[count_col].nunique().reset_index()
+    abbr_col = f'{sel_col} (first {abbr} chars)'
+    rates[abbr_col] = rates[sel_col].str[:abbr]
+    return px.line(rates.melt(id_vars=['timestamp',sel_col,abbr_col]), 
+            x='timestamp', y='value', color=abbr_col,
+            #facet_col='variable',  facet_col_wrap=1,
+            labels={'value':f'Number of Unique {count_col.title()}s per {sel_col.title()}','timestamp':''}, 
+            category_orders={abbr_col:[ v[:abbr] for v in values]},
+            # title=f'Unique {count_col.title()}s Associated with Selected {sel_col.title()}s in Metagraph',
+            title=f'Impact of Validators Update on Cabal',           
+            width=800, height=600, template='plotly_white',
+            )
+        
+        

meta_utils.py

@@ -0,0 +1,48 @@
+import os
+import glob
+import tqdm
+import pickle
+import subprocess
+import pandas as pd
+
+
+def run_subprocess(*args):
+    # Trigger the multigraph.py script to run and save metagraph snapshots
+    return subprocess.run('python multigraph.py'.split()+list(args), 
+                          shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)    
+
+def load_metagraph(path, extra_cols=None, rm_cols=None):
+    
+    with open(path, 'rb') as f:
+        metagraph = pickle.load(f)
+
+    df = pd.DataFrame(metagraph.axons)
+    df['block'] = metagraph.block.item()
+    df['difficulty'] = metagraph.difficulty
+    for c in extra_cols:
+        vals = getattr(metagraph,c)
+        df[c] = vals        
+    
+    return df.drop(columns=rm_cols)
+        
+def load_metagraphs(block_start, block_end, block_step=1000, datadir='data/metagraph/1/', extra_cols=None):
+    
+    if extra_cols is None:
+        extra_cols = ['total_stake','ranks','incentive','emission','consensus','trust','validator_trust','dividends']
+
+    blocks = range(block_start, block_end, block_step)
+    filenames = sorted(path for path in os.listdir(datadir) if int(path.split('.')[0]) in blocks)
+
+    metagraphs = []
+    
+    pbar = tqdm.tqdm(filenames)
+    for filename in pbar:
+        pbar.set_description(f'Processing {filename}')
+
+        metagraph = load_metagraph(os.path.join(datadir, filename), extra_cols=extra_cols, rm_cols=['protocol','placeholder1','placeholder2'])
+        
+        metagraphs.append(metagraph)
+        
+    return pd.concat(metagraphs)
+
+load_metagraphs(block_start=700_000, block_end=800_000, block_step=1000)

metagraph.py

@@ -0,0 +1,169 @@
+import streamlit as st
+from meta_utils import run_subprocess, load_metagraphs
+# from opendashboards.assets import io, inspect, metric, plot
+from meta_plotting import plot_trace, plot_cabals
+
+DEFAULT_SRC = 'miner'
+DEFAULT_NTOP = 10
+DEFAULT_UID_NTOP = 10
+
+# Set app config
+st.set_page_config(
+    page_title='Validator Dashboard',
+    menu_items={
+        'Report a bug': "https://github.com/opentensor/dashboards/issues",
+        'About': """
+        This dashboard is part of the OpenTensor project. \n
+        """
+    },
+    layout = "centered"
+    )
+
+st.title('Metagraph :red[Analysis] Dashboard :eyes:')
+# add vertical space
+st.markdown('#')
+st.markdown('#')
+
+
+with st.spinner(text=f'Loading data...'):
+    df = load_metagraphs()
+
+blocks = df.block.unique()
+
+# metric.wandb(df_runs)
+
+# add vertical space
+st.markdown('#')
+st.markdown('#')
+
+tab1, tab2, tab3, tab4 = st.tabs(["Health", "Miners", "Validators", "Block"])
+
+
+### Wandb Runs ###
+with tab1:
+
+    st.markdown('#')
+    st.header(":violet[Wandb] Runs")
+
+    run_msg = st.info("Select a single run or compare multiple runs")
+    selected_runs = st.multiselect(f'Runs ({len(df_runs)})', df_runs.id, default=DEFAULT_SELECTED_RUNS, key='runs')
+
+    # Load data if new runs selected
+    if not selected_runs:
+        # open a dialog to select runs
+        run_msg.error("Please select at least one run")
+        st.snow()
+        st.stop()
+
+    df = io.load_data(df_runs.loc[df_runs.id.isin(selected_runs)], load=True, save=True)
+    df_long = inspect.explode_data(df)
+    df_weights = inspect.weights(df)
+
+    metric.runs(df, df_long, selected_runs)
+
+    with st.expander(f'Show :violet[raw] data for {len(selected_runs)} selected runs'):
+        inspect.run_event_data(df_runs,df, selected_runs)
+
+
+### UID Health ###
+with tab2:
+
+    st.markdown('#')
+    st.header("UID :violet[Health]")
+    st.info(f"Showing UID health metrics for **{len(selected_runs)} selected runs**")
+
+    uid_src = st.radio('Select one:', ['followup', 'answer'], horizontal=True, key='uid_src')
+
+    metric.uids(df_long, uid_src)
+
+    with st.expander(f'Show UID **{uid_src}** weights data for **{len(selected_runs)} selected runs**'):
+
+        uids = st.multiselect('UID:', sorted(df_long[f'{uid_src}_uids'].unique()), key='uid')
+        st.markdown('#')
+        st.subheader(f"UID {uid_src.title()} :violet[Weights]")
+
+        plot.weights(
+                df_weights,
+                uids=uids,
+        )
+        
+    with st.expander(f'Show UID **{uid_src}** leaderboard data for **{len(selected_runs)} selected runs**'):
+
+        st.markdown('#')
+        st.subheader(f"UID {uid_src.title()} :violet[Leaderboard]")
+        uid_col1, uid_col2 = st.columns(2)
+        uid_ntop = uid_col1.slider('Number of UIDs:', min_value=1, max_value=50, value=DEFAULT_UID_NTOP, key='uid_ntop')
+        uid_agg = uid_col2.selectbox('Aggregation:', ('mean','min','max','size','nunique'), key='uid_agg')
+
+        plot.leaderboard(
+                df,
+                ntop=uid_ntop,
+                group_on=f'{uid_src}_uids',
+                agg_col=f'{uid_src}_rewards',
+                agg=uid_agg
+            )
+
+
+    with st.expander(f'Show UID **{uid_src}** diversity data for **{len(selected_runs)} selected runs**'):
+
+        st.markdown('#')
+        st.subheader(f"UID {uid_src.title()} :violet[Diversity]")
+        rm_failed = st.checkbox(f'Remove failed **{uid_src}** completions', value=True)
+        plot.uid_diversty(df, rm_failed)
+
+
+### Completions ###
+with tab3:
+
+    st.markdown('#')
+    st.subheader('Completion :violet[Leaderboard]')
+    completion_info = st.empty()
+
+    msg_col1, msg_col2 = st.columns(2)
+    completion_src = msg_col1.radio('Select one:', ['followup', 'answer'], horizontal=True, key='completion_src')
+    completion_info.info(f"Showing **{completion_src}** completions for **{len(selected_runs)} selected runs**")
+    
+    completion_ntop = msg_col2.slider('Top k:', min_value=1, max_value=50, value=DEFAULT_COMPLETION_NTOP, key='completion_ntop')
+
+    completion_col = f'{completion_src}_completions'
+    reward_col = f'{completion_src}_rewards'
+    uid_col = f'{completion_src}_uids'
+
+    completions = inspect.completions(df_long, completion_col)
+
+    # Get completions with highest average rewards
+    plot.leaderboard(
+        df,
+        ntop=completion_ntop,
+        group_on=completion_col,
+        agg_col=reward_col,
+        agg='mean',
+        alias=True
+    )
+
+    with st.expander(f'Show **{completion_src}** completion rewards data for **{len(selected_runs)} selected runs**'):
+
+        st.markdown('#')
+        st.subheader('Completion :violet[Rewards]')
+
+        completion_select = st.multiselect('Completions:', completions.index, default=completions.index[:3].tolist())
+        # completion_regex = st.text_input('Completion regex:', value='', key='completion_regex')
+
+        plot.completion_rewards(
+            df,
+            completion_col=completion_col,
+            reward_col=reward_col,
+            uid_col=uid_col,
+            ntop=completion_ntop,
+            completions=completion_select,
+        )
+
+
+### Prompt-based scoring ###
+with tab4:
+    # coming soon
+    st.info('Prompt-based scoring coming soon')
+
+    # st.dataframe(df_long_long.filter(regex=prompt_src).head())
+
+ 

multigraph.py

@@ -0,0 +1,112 @@
+import os
+import sys
+import argparse
+from traceback import print_exc
+import pickle
+import tqdm
+import pandas as pd
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
+
+import torch
+import bittensor
+
+#TODO: make line charts and other cool stuff for each metagraph snapshot
+
+def process(block, netuid=1, lite=True, difficulty=False, prune_weights=False, return_graph=False, half=True, subtensor=None):
+
+    if subtensor is None:
+        subtensor = bittensor.subtensor(network='finney')        
+    
+    try:        
+        metagraph = subtensor.metagraph(block=block, netuid=netuid, lite=lite)
+        if difficulty:
+            metagraph.difficulty = subtensor.difficulty(block=block, netuid=netuid)
+        
+        if not lite:
+            if half:
+                metagraph.weights = torch.nn.Parameter(metagraph.weights.half(), requires_grad=False)
+            if prune_weights:
+                metagraph.weights = metagraph.weights[metagraph.weights.sum(axis=1) > 0]
+
+        with open(f'data/metagraph/{netuid}/{block}.pkl', 'wb') as f:
+            pickle.dump(metagraph, f)
+
+        return metagraph if return_graph else True
+    
+    except Exception as e:
+        print(f'Error processing block {block}: {e}')
+        
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description='Process metagraphs for a given network.')
+    parser.add_argument('--netuid', type=int, default=1, help='Network UID to use.')
+    parser.add_argument('--difficulty', action='store_true', help='Include difficulty in metagraph.')
+    parser.add_argument('--prune_weights', action='store_true', help='Prune weights in metagraph.')
+    parser.add_argument('--return_graph', action='store_true', help='Return metagraph instead of True.')
+    parser.add_argument('--max_workers', type=int, default=32, help='Max workers to use.')
+    parser.add_argument('--start_block', type=int, default=1_000_000, help='Start block.')
+    parser.add_argument('--end_block', type=int, default=600_000, help='End block.')
+    parser.add_argument('--step_size', type=int, default=100, help='Step size.')
+    return parser.parse_args()
+
+if __name__ == '__main__':
+
+    subtensor = bittensor.subtensor(network='finney')
+    print(f'Current block: {subtensor.block}')
+    
+    args = parse_arguments()
+       
+    netuid=args.netuid
+    difficulty=args.difficulty
+    overwrite=False
+    return_graph=args.return_graph       
+     
+    step_size = args.step_size
+    start_block = args.start_block
+    start_block = (min(subtensor.block, start_block)//step_size)*step_size # round to nearest step_size
+    end_block = args.end_block
+    blocks = range(start_block, end_block, -step_size)
+
+    # only get weights for multiple of 500 blocks
+    lite=lambda x: x%500!=0
+    
+    max_workers = min(args.max_workers, len(blocks))
+
+    os.makedirs(f'data/metagraph/{netuid}', exist_ok=True)
+    if not overwrite:
+        blocks = [block for block in blocks if not os.path.exists(f'data/metagraph/{netuid}/{block}.pkl')]
+
+    metagraphs = []
+    
+    if len(blocks)==0:
+        print(f'No blocks to process. Current block: {subtensor.block}')
+        quit()
+        
+    print(f'Processing {len(blocks)} blocks from {blocks[0]}-{blocks[-1]} using {max_workers} workers.')
+    
+    with ProcessPoolExecutor(max_workers=max_workers) as executor:
+        futures = [
+            executor.submit(process, block, lite=lite(block), netuid=netuid, difficulty=difficulty) 
+            for block in blocks
+            ]
+
+        success = 0        
+        with tqdm.tqdm(total=len(futures)) as pbar:
+            for block, future in zip(blocks,futures):
+                try:
+                    metagraphs.append(future.result())
+                    success += 1
+                except Exception as e:
+                    print(f'generated an exception: {print_exc(e)}')
+                pbar.update(1)
+                pbar.set_description(f'Processed {success} blocks. Current block: {block}')
+
+    if not success:
+        raise ValueError('No blocks were successfully processed.')
+    
+    print(f'Processed {success} blocks.')
+    if return_graph:
+        for metagraph in metagraphs:
+            print(f'{metagraph.block}: {metagraph.n.item()} nodes, difficulty={getattr(metagraph, "difficulty", None)}, weights={metagraph.weights.shape if hasattr(metagraph, "weights") else None}')
+
+    print(metagraphs[-1])

multistats.py

@@ -0,0 +1,237 @@
+import os
+import warnings
+import re
+import tqdm
+import wandb
+from traceback import print_exc
+import plotly.express as px
+import pandas as pd
+from concurrent.futures import ProcessPoolExecutor
+
+import opendashboards.utils.utils as utils
+
+from IPython.display import display
+
+api= wandb.Api(timeout=60)
+wandb.login(anonymous="allow")
+
+def pull_wandb_runs(project='openvalidators', filters=None, min_steps=50, max_steps=100_000, ntop=10, summary_filters=None ):
+    # TODO: speed this up by storing older runs
+    
+    all_runs = api.runs(project, filters=filters)
+    print(f'Using {ntop}/{len(all_runs)} runs with more than {min_steps} events')
+    pbar = tqdm.tqdm(all_runs)
+    runs = []
+    n_events = 0
+    successful = 0
+    for i, run in enumerate(pbar):
+
+        summary = run.summary
+        if summary_filters is not None and not summary_filters(summary):
+            continue
+        step = summary.get('_step',0)
+        if step < min_steps or step > max_steps:
+            # warnings.warn(f'Skipped run `{run.name}` because it contains {step} events (<{min_steps})')
+            continue
+
+        prog_msg = f'Loading data {i/len(all_runs)*100:.0f}% ({successful}/{len(all_runs)} runs, {n_events} events)'
+        pbar.set_description(f'{prog_msg}... **fetching** `{run.name}`')
+
+        duration = summary.get('_runtime')
+        end_time = summary.get('_timestamp')
+        # extract values for selected tags
+        rules = {'hotkey': re.compile('^[0-9a-z]{48}$',re.IGNORECASE), 'version': re.compile('^\\d\.\\d+\.\\d+$'), 'spec_version': re.compile('\\d{4}$')}
+        tags = {k: tag for k, rule in rules.items() for tag in run.tags if rule.match(tag)}
+        # include bool flag for remaining tags
+        tags.update({k: True for k in run.tags if k not in tags.keys() and k not in tags.values()})
+
+        runs.append({
+            'state': run.state,
+            'num_steps': step,
+            'num_completions': step*sum(len(v) for k, v in run.summary.items() if k.endswith('completions') and isinstance(v, list)),
+            'entity': run.entity,
+            'user': run.user.name,
+            'username': run.user.username,
+            'run_id': run.id,
+            'run_name': run.name,
+            'project': run.project,
+            'run_url': run.url,
+            'run_path': os.path.join(run.entity, run.project, run.id),
+            'start_time': pd.to_datetime(end_time-duration, unit="s"),
+            'end_time': pd.to_datetime(end_time, unit="s"),
+            'duration': pd.to_timedelta(duration, unit="s").round('s'),
+            **tags
+        })
+        n_events += step
+        successful += 1
+        if successful >= ntop:
+            break
+
+    return pd.DataFrame(runs).astype({'state': 'category', 'hotkey': 'category', 'version': 'category', 'spec_version': 'category'})
+
+def plot_gantt(df_runs):
+    fig = px.timeline(df_runs,
+                x_start="start_time", x_end="end_time", y="username", color="state",
+                title="Timeline of Runs",
+                category_orders={'run_name': df_runs.run_name.unique()},#,'username': sorted(df_runs.username.unique())},
+                hover_name="run_name",
+                hover_data=['hotkey','user','username','run_id','num_steps','num_completions'],
+                color_discrete_map={'running': 'green', 'finished': 'grey', 'killed':'blue', 'crashed':'orange', 'failed': 'red'},
+                opacity=0.3,
+                width=1200,
+                height=800,
+                template="plotly_white",
+    )
+    fig.update_yaxes(tickfont_size=8, title='')
+    fig.show()
+
+def load_data(run_id, run_path=None, load=True, save=False, timeout=30):
+
+    file_path = os.path.join('data/runs/',f'history-{run_id}.csv')
+
+    if load and os.path.exists(file_path):
+        df = pd.read_csv(file_path, nrows=None)
+        # filter out events with missing step length
+        df = df.loc[df.step_length.notna()]
+
+        # detect list columns which as stored as strings
+        list_cols = [c for c in df.columns if df[c].dtype == "object" and df[c].str.startswith("[").all()]
+        # convert string representation of list to list
+        df[list_cols] = df[list_cols].applymap(eval, na_action='ignore')
+
+    else:
+        # Download the history from wandb and add metadata
+        run = api.run(run_path)
+        df = pd.DataFrame(list(run.scan_history()))
+
+        print(f'Downloaded {df.shape[0]} events from {run_path!r} with id {run_id!r}')
+
+        if save:
+            df.to_csv(file_path, index=False)
+
+    # Convert timestamp to datetime.
+    df._timestamp = pd.to_datetime(df._timestamp, unit="s")
+    return df.sort_values("_timestamp")
+
+
+def calculate_stats(df_long, rm_failed=True, rm_zero_reward=True, freq='H', save_path=None ):
+
+    df_long._timestamp = pd.to_datetime(df_long._timestamp)
+    # if dataframe has columns such as followup_completions and answer_completions, convert to multiple rows
+    if 'completions' not in df_long.columns:
+        df_long.set_index(['_timestamp','run_id'], inplace=True)
+        df_schema = pd.concat([
+            df_long[['followup_completions','followup_rewards']].rename(columns={'followup_completions':'completions', 'followup_rewards':'rewards'}),
+            df_long[['answer_completions','answer_rewards']].rename(columns={'answer_completions':'completions', 'answer_rewards':'rewards'})
+        ])
+        df_long = df_schema.reset_index()
+
+    if rm_failed:
+        df_long = df_long.loc[ df_long.completions.str.len()>0 ]
+
+    if rm_zero_reward:
+        df_long = df_long.loc[ df_long.rewards>0 ]
+
+    print(f'Calculating stats for dataframe with shape {df_long.shape}')
+
+    g = df_long.groupby([pd.Grouper(key='_timestamp', axis=0, freq=freq), 'run_id'])
+
+    stats = g.agg({'completions':['nunique','count'], 'rewards':['sum','mean','std']})
+
+    stats.columns = ['_'.join(c) for c in stats.columns]
+    stats['completions_diversity'] = stats['completions_nunique'] / stats['completions_count']
+    stats = stats.reset_index()
+
+    if save_path:        
+        stats.to_csv(save_path, index=False)
+
+    return stats
+
+
+def clean_data(df):
+    return df.dropna(subset=df.filter(regex='completions|rewards').columns, how='any').dropna(axis=1, how='all')
+
+def explode_data(df):
+    list_cols = utils.get_list_col_lengths(df)
+    return utils.explode_data(df, list(list_cols.keys())).apply(pd.to_numeric, errors='ignore')
+
+
+def process(run, load=True, save=False, freq='H'):
+
+    try:
+      
+        stats_path = f'data/aggs/stats-{run["run_id"]}.csv'
+        if os.path.exists(stats_path):
+            print(f'Loaded stats file {stats_path}')
+            return pd.read_csv(stats_path)
+
+        # Load data and add extra columns from wandb run
+        df = load_data(run_id=run['run_id'],
+                    run_path=run['run_path'],
+                    load=load,
+                    save=save, 
+                    save = (run['state'] != 'running') & run['end_time']
+                    ).assign(**run.to_dict())
+        # Clean and explode dataframe
+        df_long = explode_data(clean_data(df))
+        # Remove original dataframe from memory
+        del df
+        # Get and save stats
+        return calculate_stats(df_long, freq=freq, save_path=stats_path)
+    
+    except Exception as e:
+        print(f'Error processing run {run["run_id"]}: {e}')
+
+if __name__ == '__main__':
+
+    # TODO: flag to overwrite runs that were running when downloaded and saved: check if file date is older than run end time.
+    
+    filters = None# {"tags": {"$in": [f'1.1.{i}' for i in range(10)]}}
+    # filters={'tags': {'$in': ['5F4tQyWrhfGVcNhoqeiNsR6KjD4wMZ2kfhLj4oHYuyHbZAc3']}} # Is foundation validator
+    df_runs = pull_wandb_runs(ntop=500, filters=filters)#summary_filters=lambda s: s.get('augment_prompt'))
+
+    os.makedirs('data/runs/', exist_ok=True)
+    os.makedirs('data/aggs/', exist_ok=True)
+    df_runs.to_csv('data/wandb.csv', index=False)
+    
+    display(df_runs)
+    plot_gantt(df_runs)
+
+    with ProcessPoolExecutor(max_workers=min(32, df_runs.shape[0])) as executor:
+        futures = [executor.submit(process, run, load=True, save=True) for _, run in df_runs.iterrows()]
+
+        # Use tqdm to add a progress bar
+        results = []
+        with tqdm.tqdm(total=len(futures)) as pbar:
+            for future in futures:
+                try:
+                    result = future.result()
+                    results.append(result)
+                except Exception as e:
+                    print(f'generated an exception: {print_exc(e)}')
+                pbar.update(1)
+
+    if not results:
+        raise ValueError('No runs were successfully processed.')
+
+   # Concatenate the results into a single dataframe
+    df = pd.concat(results, ignore_index=True)
+
+    df.to_csv('data/processed.csv', index=False)
+
+    display(df)
+
+    fig = px.line(df.astype({'_timestamp':str}),
+              x='_timestamp',
+              y='completions_diversity',
+            #   y=['Unique','Total'],
+        line_group='run_id',
+        # color='hotkey',
+        # color_discrete_sequence=px.colors.sequential.YlGnBu,
+        title='Completion Diversity over Time',
+        labels={'_timestamp':'', 'completions_diversity':'Diversity', 'uids':'UID','value':'counts', 'variable':'Completions'},
+        width=800, height=600,
+        template='plotly_white',
+        ).update_traces(opacity=0.3)
+    fig.show()
+