Merge pull request #4 from opentensor/aggregations

dashboard.py

@@ -1,13 +1,16 @@
+import time
+import pandas as pd
 import streamlit as st
 from opendashboards.assets import io, inspect, metric, plot
 
-# dendrite time versus completion length
 # prompt-based completion score stats
 # instrospect specific RUN-UID-COMPLETION
+# cache individual file loads
+# Hotkey churn
 
 DEFAULT_PROJECT = "openvalidators"
-DEFAULT_FILTERS = {"tags": {"$in": ["1.0.0", "1.0.1", "1.0.2", "1.0.3", "1.0.4"]}}
-DEFAULT_SELECTED_RUNS = ['kt9bzxii']
+DEFAULT_FILTERS = {"tags": {"$in": [f'1.1.{i}' for i in range(10)]}}
+DEFAULT_SELECTED_HOTKEYS = None
 DEFAULT_SRC = 'followup'
 DEFAULT_COMPLETION_NTOP = 10
 DEFAULT_UID_NTOP = 10
@@ -31,8 +34,6 @@ st.title('Validator :red[Analysis] Dashboard :eyes:')
 st.markdown('#')
 st.markdown('#')
 
-# with st.sidebar:
-#     st.sidebar.header('Pages')
 
 with st.spinner(text=f'Checking wandb...'):
     df_runs = io.load_runs(project=DEFAULT_PROJECT, filters=DEFAULT_FILTERS, min_steps=10)
@@ -43,58 +44,83 @@ metric.wandb(df_runs)
 st.markdown('#')
 st.markdown('#')
 
-tab1, tab2, tab3, tab4 = st.tabs(["Wandb Runs", "UID Health", "Completions", "Prompt-based scoring"])
-
+tab1, tab2, tab3, tab4 = st.tabs(["Raw Data", "UID Health", "Completions", "Prompt-based scoring"])
 
 ### Wandb Runs ###
 with tab1:
 
     st.markdown('#')
-    st.header(":violet[Wandb] Runs")
-
-    run_msg = st.info("Select a single run or compare multiple runs")
-    selected_runs = st.multiselect(f'Runs ({len(df_runs)})', df_runs.id, default=DEFAULT_SELECTED_RUNS, key='runs')
-
-    # Load data if new runs selected
-    if not selected_runs:
-        # open a dialog to select runs
-        run_msg.error("Please select at least one run")
-        st.snow()
+    st.subheader(":violet[Run] Data")
+    with st.expander(f'Show :violet[raw] wandb data'):
+
+        edited_df = st.data_editor(
+            df_runs.assign(Select=False).set_index('Select'),
+            column_config={"Select": st.column_config.CheckboxColumn(required=True)},
+            disabled=df_runs.columns,
+            use_container_width=True,
+        )
+        df_runs_subset = df_runs[edited_df.index==True]
+        n_runs = len(df_runs_subset)
+
+    if n_runs:
+        df = io.load_data(df_runs_subset, load=True, save=True)
+        df = inspect.clean_data(df)
+        print(f'\nNans in columns: {df.isna().sum()}')
+        df_long = inspect.explode_data(df)
+    else:
+        st.info(f'You must select at least one run to load data')
         st.stop()
 
-    df = io.load_data(df_runs.loc[df_runs.id.isin(selected_runs)], load=True, save=True)
-    df_long = inspect.explode_data(df)
-    df_weights = inspect.weights(df)
-
-    metric.runs(df, df_long, selected_runs)
+    metric.runs(df_long)
 
-    with st.expander(f'Show :violet[raw] data for {len(selected_runs)} selected runs'):
-        inspect.run_event_data(df_runs,df, selected_runs)
+    st.markdown('#')
+    st.subheader(":violet[Event] Data")
+    with st.expander(f'Show :violet[raw] event data for **{n_runs} selected runs**'):
+        raw_data_col1, raw_data_col2 = st.columns(2)
+        use_long_checkbox = raw_data_col1.checkbox('Use long format', value=True)
+        num_rows = raw_data_col2.slider('Number of rows:', min_value=1, max_value=100, value=10, key='num_rows')
+        st.dataframe(df_long.head(num_rows) if use_long_checkbox else df.head(num_rows),
+                     use_container_width=True)
 
+step_types = ['all']+['augment','followup','answer']#list(df.name.unique())
 
 ### UID Health ###
+# TODO: Live time - time elapsed since moving_averaged_score for selected UID was 0 (lower bound so use >Time)
+# TODO: Weight - Most recent weight for selected UID (Add warning if weight is 0 or most recent timestamp is not current)
 with tab2:
 
     st.markdown('#')
-    st.header("UID :violet[Health]")
-    st.info(f"Showing UID health metrics for **{len(selected_runs)} selected runs**")
+    st.subheader("UID :violet[Health]")
+    st.info(f"Showing UID health metrics for **{n_runs} selected runs**")
 
-    uid_src = st.radio('Select one:', ['followup', 'answer'], horizontal=True, key='uid_src')
+    uid_src = st.radio('Select event type:', step_types, horizontal=True, key='uid_src')
+    df_uid = df_long[df_long.name.str.contains(uid_src)] if uid_src != 'all' else df_long
+        
+    metric.uids(df_uid, uid_src)
+    uids = st.multiselect('UID:', sorted(df_uid['uids'].unique()), key='uid')
+    with st.expander(f'Show UID health data for **{n_runs} selected runs** and **{len(uids)} selected UIDs**'):
+        st.markdown('#')
+        st.subheader(f"UID {uid_src.title()} :violet[Health]")
+        agg_uid_checkbox = st.checkbox('Aggregate UIDs', value=True)
+        if agg_uid_checkbox:
+            metric.uids(df_uid, uid_src, uids)
+        else:
+            for uid in uids:
+                st.caption(f'UID: {uid}')
+                metric.uids(df_uid, uid_src, [uid])
 
-    metric.uids(df_long, uid_src)
+        st.subheader(f'Cumulative completion frequency')
 
-    with st.expander(f'Show UID **{uid_src}** weights data for **{len(selected_runs)} selected runs**'):
+        freq_col1, freq_col2 = st.columns(2)
+        freq_ntop = freq_col1.slider('Number of Completions:', min_value=10, max_value=1000, value=100, key='freq_ntop')
+        freq_rm_empty = freq_col2.checkbox('Remove empty (failed)', value=True, key='freq_rm_empty')
+        freq_cumulative = freq_col2.checkbox('Cumulative', value=False, key='freq_cumulative')
+        freq_normalize = freq_col2.checkbox('Normalize', value=True, key='freq_normalize')
 
-        uids = st.multiselect('UID:', sorted(df_long[f'{uid_src}_uids'].unique()), key='uid')
-        st.markdown('#')
-        st.subheader(f"UID {uid_src.title()} :violet[Weights]")
+        plot.uid_completion_counts(df_uid, uids=uids, src=uid_src, ntop=freq_ntop, rm_empty=freq_rm_empty, cumulative=freq_cumulative, normalize=freq_normalize)
 
-        plot.weights(
-                df_weights,
-                uids=uids,
-        )
-        
-    with st.expander(f'Show UID **{uid_src}** leaderboard data for **{len(selected_runs)} selected runs**'):
+
+    with st.expander(f'Show UID **{uid_src}** leaderboard data for **{n_runs} selected runs**'):
 
         st.markdown('#')
         st.subheader(f"UID {uid_src.title()} :violet[Leaderboard]")
@@ -103,15 +129,15 @@ with tab2:
         uid_agg = uid_col2.selectbox('Aggregation:', ('mean','min','max','size','nunique'), key='uid_agg')
 
         plot.leaderboard(
-                df,
+                df_uid,
                 ntop=uid_ntop,
-                group_on=f'{uid_src}_uids',
-                agg_col=f'{uid_src}_rewards',
+                group_on='uids',
+                agg_col='rewards',
                 agg=uid_agg
             )
 
 
-    with st.expander(f'Show UID **{uid_src}** diversity data for **{len(selected_runs)} selected runs**'):
+    with st.expander(f'Show UID **{uid_src}** diversity data for **{n_runs} selected runs**'):
 
         st.markdown('#')
         st.subheader(f"UID {uid_src.title()} :violet[Diversity]")
@@ -127,28 +153,27 @@ with tab3:
     completion_info = st.empty()
 
     msg_col1, msg_col2 = st.columns(2)
-    completion_src = msg_col1.radio('Select one:', ['followup', 'answer'], horizontal=True, key='completion_src')
-    completion_info.info(f"Showing **{completion_src}** completions for **{len(selected_runs)} selected runs**")
+    # completion_src = msg_col1.radio('Select one:', ['followup', 'answer'], horizontal=True, key='completion_src')
+    completion_src = st.radio('Select event type:', step_types, horizontal=True, key='completion_src')
+    df_comp = df_long[df_long.name.str.contains(completion_src)] if completion_src != 'all' else df_long
     
-    completion_ntop = msg_col2.slider('Top k:', min_value=1, max_value=50, value=DEFAULT_COMPLETION_NTOP, key='completion_ntop')
+    completion_info.info(f"Showing **{completion_src}** completions for **{n_runs} selected runs**")
 
-    completion_col = f'{completion_src}_completions'
-    reward_col = f'{completion_src}_rewards'
-    uid_col = f'{completion_src}_uids'
+    completion_ntop = msg_col2.slider('Top k:', min_value=1, max_value=50, value=DEFAULT_COMPLETION_NTOP, key='completion_ntop')
 
-    completions = inspect.completions(df_long, completion_col)
+    completions = inspect.completions(df_long, 'completions')
 
     # Get completions with highest average rewards
     plot.leaderboard(
-        df,
+        df_comp,
         ntop=completion_ntop,
-        group_on=completion_col,
-        agg_col=reward_col,
+        group_on='completions',
+        agg_col='rewards',
         agg='mean',
         alias=True
     )
 
-    with st.expander(f'Show **{completion_src}** completion rewards data for **{len(selected_runs)} selected runs**'):
+    with st.expander(f'Show **{completion_src}** completion rewards data for **{n_runs} selected runs**'):
 
         st.markdown('#')
         st.subheader('Completion :violet[Rewards]')
@@ -157,19 +182,37 @@ with tab3:
         # completion_regex = st.text_input('Completion regex:', value='', key='completion_regex')
 
         plot.completion_rewards(
-            df,
-            completion_col=completion_col,
-            reward_col=reward_col,
-            uid_col=uid_col,
+            df_comp,
+            completion_col='completions',
+            reward_col='rewards',
+            uid_col='uids',
             ntop=completion_ntop,
             completions=completion_select,
         )
+        # TODO: show the UIDs which have used the selected completions
+
 
+    with st.expander(f'Show **{completion_src}** completion length data for **{n_runs} selected runs**'):
+
+        st.markdown('#')
+        st.subheader('Completion :violet[Length]')
+
+        completion_length_radio = st.radio('Use: ', ['characters','words','sentences'], key='completion_length_radio')
+
+        # Todo: use color to identify selected completions/ step names/ uids
+        plot.completion_length_time(
+            df_comp,
+            completion_col='completions',
+            uid_col='uids',
+            time_col='completion_times',
+            length_opt=completion_length_radio,
+        )
 
 ### Prompt-based scoring ###
 with tab4:
     # coming soon
     st.info('Prompt-based scoring coming soon')
+    st.snow()
 
     # st.dataframe(df_long_long.filter(regex=prompt_src).head())
 

meta_plotting.py

@@ -0,0 +1,48 @@
+import numpy as np
+import plotly.express as px
+
+def plot_trace(df, col='emission', agg='mean', time_col='timestamp', ntop=10, hotkeys=None, hotkey_regex=None, abbrev=8, type='Miners'):
+    
+    if hotkeys is not None:
+        df = df.loc[df.hotkey.isin(hotkeys)]
+    if hotkey_regex is not None:
+        df = df.loc[df.hotkey.str.contains(hotkey_regex)]
+        
+    top_miners = df.groupby('hotkey')[col].agg(agg).sort_values(ascending=False)
+    
+    stats = df.loc[df.hotkey.isin(top_miners.index[:ntop])].sort_values(by=time_col)
+    
+    stats['hotkey_abbrev'] = stats.hotkey.str[:abbrev]
+    stats['coldkey_abbrev'] = stats.coldkey.str[:abbrev]
+    stats['rank'] = stats.hotkey.map({k:i for i,k in enumerate(top_miners.index, start=1)})
+    
+    return px.line(stats.sort_values(by=[time_col,'rank']),
+                    x=time_col, y=col, color='coldkey_abbrev', line_group='hotkey_abbrev',
+                    hover_data=['hotkey','rank'],
+                    labels={col:col.title(),'timestamp':'','coldkey_abbrev':f'Coldkey (first {abbrev} chars)','hotkey_abbrev':f'Hotkey (first {abbrev} chars)'},
+                    title=f'Top {ntop} {type}, by {col.title()}',
+                    template='plotly_white', width=800, height=600,
+                    ).update_traces(opacity=0.7)
+    
+    
+def plot_cabals(df, sel_col='coldkey', count_col='hotkey', time_col='timestamp', values=None, ntop=10, abbr=8):
+    
+    if values is None:
+        values = df[sel_col].value_counts().sort_values(ascending=False).index[:ntop].tolist()
+        print(f'Automatically selected {sel_col!r} = {values!r}')
+        
+    df = df.loc[df[sel_col].isin(values)]
+    rates = df.groupby([time_col,sel_col])[count_col].nunique().reset_index()
+    abbr_col = f'{sel_col} (first {abbr} chars)'
+    rates[abbr_col] = rates[sel_col].str[:abbr]
+    return px.line(rates.melt(id_vars=[time_col,sel_col,abbr_col]), 
+            x=time_col, y='value', color=abbr_col,
+            #facet_col='variable',  facet_col_wrap=1,
+            labels={'value':f'Number of Unique {count_col.title()}s per {sel_col.title()}','timestamp':''}, 
+            category_orders={abbr_col:[ v[:abbr] for v in values]},
+            # title=f'Unique {count_col.title()}s Associated with Selected {sel_col.title()}s in Metagraph',
+            title=f'Impact of Validators Update on Cabal',           
+            width=800, height=600, template='plotly_white',
+            )
+        
+        

meta_utils.py

@@ -0,0 +1,59 @@
+import os
+import glob
+import tqdm
+import dill as pickle
+import subprocess
+import pandas as pd
+import datetime
+from functools import lru_cache
+
+block_time_500k = datetime.datetime(2023, 5, 29, 5, 29, 0)
+block_time_800k = datetime.datetime(2023, 7, 9, 21, 32, 48)
+dt = (pd.Timestamp(block_time_800k)-pd.Timestamp(block_time_500k))/(800_000-500_000)
+
+def run_subprocess(*args):
+    # Trigger the multigraph.py script to run and save metagraph snapshots
+    return subprocess.run('python multigraph.py'.split()+list(args), 
+                          shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)    
+
+def load_metagraph(path, extra_cols=None, rm_cols=None):
+    
+    with open(path, 'rb') as f:
+        metagraph = pickle.load(f)
+
+    df = pd.DataFrame(metagraph.axons)
+    df['block'] = metagraph.block.item()
+    df['timestamp'] = block_time_500k + dt*(df['block']-500_000)
+    df['difficulty'] = metagraph.difficulty
+    for c in extra_cols:
+        vals = getattr(metagraph,c)
+        df[c] = vals        
+    
+    return df.drop(columns=rm_cols)
+
+@lru_cache(maxsize=16)
+def load_metagraphs(block_start, block_end, block_step=1000, datadir='data/metagraph/1/', extra_cols=None):
+    
+    if extra_cols is None:
+        extra_cols = ['total_stake','ranks','incentive','emission','consensus','trust','validator_trust','dividends']
+
+    blocks = range(block_start, block_end, block_step)
+    print(f'Loading blocks {blocks[0]}-{blocks[-1]} from {datadir}')
+    filenames = sorted(filename for filename in os.listdir(datadir) if int(filename.split('.')[0]) in blocks)
+    print(f'Found {len(filenames)} files in {datadir}')
+    
+    metagraphs = []
+    
+    pbar = tqdm.tqdm(filenames)
+    for filename in pbar:
+        pbar.set_description(f'Processing {filename}')
+
+        try:
+            metagraph = load_metagraph(os.path.join(datadir, filename), extra_cols=extra_cols, rm_cols=['protocol','placeholder1','placeholder2'])
+            
+            metagraphs.append(metagraph)
+        except Exception as e:
+            print(f'filename {filename!r} generated an exception: { e }')
+            
+    return pd.concat(metagraphs)
+

metadash.py

@@ -0,0 +1,114 @@
+import os
+import pandas as pd
+import streamlit as st
+from meta_utils import run_subprocess, load_metagraphs
+# from opendashboards.assets import io, inspect, metric, plot
+from meta_plotting import plot_trace, plot_cabals
+import asyncio
+
+## TODO: Read blocks from a big parquet file instead of loading all the pickles -- this is slow
+
+def get_or_create_eventloop():
+    try:
+        return asyncio.get_event_loop()
+    except RuntimeError as ex:
+        if "There is no current event loop in thread" in str(ex):
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            return asyncio.get_event_loop()
+
+loop = asyncio.new_event_loop()
+asyncio.set_event_loop(loop)
+import bittensor
+
+datadir='data/metagraph/1/'
+blockfiles = sorted(int(filename.split('.')[0]) for filename in os.listdir(datadir))
+DEFAULT_SRC = 'miner'
+DEFAULT_BLOCK_START = blockfiles[0]
+DEFAULT_BLOCK_END = blockfiles[-1]
+DEFAULT_BLOCK_STEP = 1000
+DEFAULT_NTOP = 10
+DEFAULT_UID_NTOP = 10
+
+# Set app config
+st.set_page_config(
+    page_title='Validator Dashboard',
+    menu_items={
+        'Report a bug': "https://github.com/opentensor/dashboards/issues",
+        'About': """
+        This dashboard is part of the OpenTensor project. \n
+        """
+    },
+    layout = "centered"
+    )
+
+st.title('Metagraph :red[Analysis] Dashboard :eyes:')
+# add vertical space
+st.markdown('#')
+st.markdown('#')
+
+subtensor = bittensor.subtensor(network='finney')
+current_block = subtensor.get_current_block()
+current_difficulty = subtensor.difficulty(1, block=current_block)
+
+bcol1, bcol2, bcol3 = st.columns([0.2, 0.6, 0.2])
+with bcol1:
+    st.metric('Current **block**', current_block, delta='+7200 [24hr]')
+    # st.metric('Current **difficulty**', f'{current_difficulty/10e12:.0}T', delta='?')
+    
+    
+block_start, block_end = bcol2.select_slider(
+    'Select a **block range**',
+    options=blockfiles,
+    value=(DEFAULT_BLOCK_START, DEFAULT_BLOCK_END),
+    format_func=lambda x: f'{x:,}'
+)
+
+bcol3.button('Refresh', on_click=run_subprocess)
+
+
+with st.spinner(text=f'Loading data...'):
+    # df = load_metagraphs(block_start=block_start, block_end=block_end, block_step=DEFAULT_BLOCK_STEP)
+    df = pd.read_parquet('blocks_600100_807300_100')
+
+blocks = df.block.unique()
+
+df_sel = df.loc[df.block.between(block_start, block_end)]
+
+
+# add vertical space
+st.markdown('#')
+st.markdown('#')
+
+tab1, tab2, tab3, tab4 = st.tabs(["Overview", "Miners", "Validators", "Block"])
+
+miner_choices = ['total_stake','ranks','incentive','emission','consensus','trust','validator_trust','dividends']
+cabal_choices = ['hotkey','ip','coldkey']
+
+### Overview  ###
+with tab1:
+    
+    x_col = st.radio('X-axis', ['block','timestamp'], index=0, horizontal=True)
+    
+    acol1, acol2 = st.columns([0.3, 0.7])
+    sel_ntop = acol1.slider('Number:', min_value=1, max_value=50, value=10, key='sel_ntop')    
+    #horizontal list
+    miner_choice = acol2.radio('Select:', miner_choices, horizontal=True, index=0)
+    st.plotly_chart(
+        plot_trace(df_sel, time_col=x_col,col=miner_choice, ntop=sel_ntop),
+        use_container_width=True
+    )
+
+    col1, col2 = st.columns(2)
+    count_col = col1.radio('Count', cabal_choices, index=0, horizontal=True)
+    y_col = col2.radio('Agg on', cabal_choices, index=2, horizontal=True)
+
+    st.plotly_chart(
+        plot_cabals(df_sel, time_col=x_col, count_col=count_col, sel_col=y_col, ntop=sel_ntop), 
+        use_container_width=True
+    )
+
+with tab2:
+    
+    # plot of miner weights versus time/block
+    pass

multigraph.py

@@ -0,0 +1,112 @@
+import os
+import sys
+import argparse
+from traceback import print_exc
+import pickle
+import tqdm
+import pandas as pd
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
+
+import torch
+import bittensor
+
+#TODO: make line charts and other cool stuff for each metagraph snapshot
+
+def process(block, netuid=1, lite=True, difficulty=False, prune_weights=False, return_graph=False, half=True, subtensor=None):
+
+    if subtensor is None:
+        subtensor = bittensor.subtensor(network='finney')        
+    
+    try:        
+        metagraph = subtensor.metagraph(block=block, netuid=netuid, lite=lite)
+        if difficulty:
+            metagraph.difficulty = subtensor.difficulty(block=block, netuid=netuid)
+        
+        if not lite:
+            if half:
+                metagraph.weights = torch.nn.Parameter(metagraph.weights.half(), requires_grad=False)
+            if prune_weights:
+                metagraph.weights = metagraph.weights[metagraph.weights.sum(axis=1) > 0]
+
+        with open(f'data/metagraph/{netuid}/{block}.pkl', 'wb') as f:
+            pickle.dump(metagraph, f)
+
+        return metagraph if return_graph else True
+    
+    except Exception as e:
+        print(f'Error processing block {block}: {e}')
+        
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description='Process metagraphs for a given network.')
+    parser.add_argument('--netuid', type=int, default=1, help='Network UID to use.')
+    parser.add_argument('--difficulty', action='store_true', help='Include difficulty in metagraph.')
+    parser.add_argument('--prune_weights', action='store_true', help='Prune weights in metagraph.')
+    parser.add_argument('--return_graph', action='store_true', help='Return metagraph instead of True.')
+    parser.add_argument('--max_workers', type=int, default=32, help='Max workers to use.')
+    parser.add_argument('--start_block', type=int, default=1_000_000, help='Start block.')
+    parser.add_argument('--end_block', type=int, default=600_000, help='End block.')
+    parser.add_argument('--step_size', type=int, default=100, help='Step size.')
+    return parser.parse_args()
+
+if __name__ == '__main__':
+
+    subtensor = bittensor.subtensor(network='finney')
+    print(f'Current block: {subtensor.block}')
+    
+    args = parse_arguments()
+       
+    netuid=args.netuid
+    difficulty=args.difficulty
+    overwrite=False
+    return_graph=args.return_graph       
+     
+    step_size = args.step_size
+    start_block = args.start_block
+    start_block = (min(subtensor.block, start_block)//step_size)*step_size # round to nearest step_size
+    end_block = args.end_block
+    blocks = range(start_block, end_block, -step_size)
+
+    # only get weights for multiple of 500 blocks
+    lite=lambda x: x%500!=0
+    
+    max_workers = min(args.max_workers, len(blocks))
+
+    os.makedirs(f'data/metagraph/{netuid}', exist_ok=True)
+    if not overwrite:
+        blocks = [block for block in blocks if not os.path.exists(f'data/metagraph/{netuid}/{block}.pkl')]
+
+    metagraphs = []
+    
+    if len(blocks)==0:
+        print(f'No blocks to process. Current block: {subtensor.block}')
+        quit()
+        
+    print(f'Processing {len(blocks)} blocks from {blocks[0]}-{blocks[-1]} using {max_workers} workers.')
+    
+    with ProcessPoolExecutor(max_workers=max_workers) as executor:
+        futures = [
+            executor.submit(process, block, lite=lite(block), netuid=netuid, difficulty=difficulty) 
+            for block in blocks
+            ]
+
+        success = 0        
+        with tqdm.tqdm(total=len(futures)) as pbar:
+            for block, future in zip(blocks,futures):
+                try:
+                    metagraphs.append(future.result())
+                    success += 1
+                except Exception as e:
+                    print(f'generated an exception: {print_exc(e)}')
+                pbar.update(1)
+                pbar.set_description(f'Processed {success} blocks. Current block: {block}')
+
+    if not success:
+        raise ValueError('No blocks were successfully processed.')
+    
+    print(f'Processed {success} blocks.')
+    if return_graph:
+        for metagraph in metagraphs:
+            print(f'{metagraph.block}: {metagraph.n.item()} nodes, difficulty={getattr(metagraph, "difficulty", None)}, weights={metagraph.weights.shape if hasattr(metagraph, "weights") else None}')
+
+    print(metagraphs[-1])

multistats.py

@@ -0,0 +1,348 @@
+import os
+import re
+import argparse
+import tqdm
+import wandb
+import traceback
+import plotly.express as px
+import pandas as pd
+from concurrent.futures import ProcessPoolExecutor
+
+import opendashboards.utils.utils as utils
+import opendashboards.utils.aggregate as aggregate
+
+from IPython.display import display
+
+api= wandb.Api(timeout=60)
+wandb.login(anonymous="allow")
+
+def pull_wandb_runs(project='openvalidators', filters=None, min_steps=50, max_steps=100_000, ntop=10, netuid=None, summary_filters=None ):
+    # TODO: speed this up by storing older runs
+
+    all_runs = api.runs(project, filters=filters)
+    print(f'Using {ntop}/{len(all_runs)} runs with more than {min_steps} events')
+    pbar = tqdm.tqdm(all_runs)
+    runs = []
+    n_events = 0
+    successful = 0
+    for i, run in enumerate(pbar):
+
+        summary = run.summary
+        if summary_filters is not None and not summary_filters(summary):
+            continue
+        if netuid is not None and run.config.get('netuid') != netuid:
+            continue
+        step = summary.get('_step',0)
+        if step < min_steps or step > max_steps:
+            # warnings.warn(f'Skipped run `{run.name}` because it contains {step} events (<{min_steps})')
+            continue
+
+        prog_msg = f'Loading data {successful/ntop*100:.0f}% ({successful}/{ntop} runs, {n_events} events)'
+        pbar.set_description(f'{prog_msg}... **fetching** `{run.name}`')
+
+        duration = summary.get('_runtime')
+        end_time = summary.get('_timestamp')
+        # extract values for selected tags
+        rules = {'hotkey': re.compile('^[0-9a-z]{48}$',re.IGNORECASE), 'version': re.compile('^\\d\.\\d+\.\\d+$'), 'spec_version': re.compile('\\d{4}$')}
+        tags = {k: tag for k, rule in rules.items() for tag in run.tags if rule.match(tag)}
+        # include bool flag for remaining tags
+        tags.update({k: True for k in run.tags if k not in tags.keys() and k not in tags.values()})
+
+        runs.append({
+            'state': run.state,
+            'num_steps': step,
+            'num_completions': step*sum(len(v) for k, v in run.summary.items() if k.endswith('completions') and isinstance(v, list)),
+            'entity': run.entity,
+            'user': run.user.name,
+            'username': run.user.username,
+            'run_id': run.id,
+            'run_name': run.name,
+            'project': run.project,
+            'run_url': run.url,
+            'run_path': os.path.join(run.entity, run.project, run.id),
+            'start_time': pd.to_datetime(end_time-duration, unit="s"),
+            'end_time': pd.to_datetime(end_time, unit="s"),
+            'duration': pd.to_timedelta(duration, unit="s").round('s'),
+            'netuid': run.config.get('netuid'),
+            **tags
+        })
+        n_events += step
+        successful += 1
+        if successful >= ntop:
+            break
+
+    return pd.DataFrame(runs).astype({'state': 'category', 'hotkey': 'category', 'version': 'category', 'spec_version': 'category'})
+
+def plot_gantt(df_runs):
+    fig = px.timeline(df_runs,
+                x_start="start_time", x_end="end_time", y="username", color="state",
+                title="Timeline of Runs",
+                category_orders={'run_name': df_runs.run_name.unique()},#,'username': sorted(df_runs.username.unique())},
+                hover_name="run_name",
+                hover_data=['hotkey','user','username','run_id','num_steps','num_completions'],
+                color_discrete_map={'running': 'green', 'finished': 'grey', 'killed':'blue', 'crashed':'orange', 'failed': 'red'},
+                opacity=0.3,
+                width=1200,
+                height=800,
+                template="plotly_white",
+    )
+    fig.update_yaxes(tickfont_size=8, title='')
+    fig.show()
+
+
+def clean_data(df):
+    return df.dropna(subset=df.filter(regex='completions|rewards').columns, how='any').dropna(axis=1, how='all')
+
+def explode_data(df):
+    list_cols = utils.get_list_col_lengths(df)
+    return utils.explode_data(df, list(list_cols.keys())).apply(pd.to_numeric, errors='ignore')
+
+
+def load_data(run_id, run_path=None, load=True, save=False, explode=True):
+
+    file_path = os.path.join('data/runs/',f'history-{run_id}.parquet')
+
+    if load and os.path.exists(file_path):
+        df = pd.read_parquet(file_path)
+        # filter out events with missing step length
+        df = df.loc[df.step_length.notna()]
+
+        # detect list columns which as stored as strings
+        ignore_cols = ('moving_averaged_scores')
+        list_cols = [c for c in df.columns if c not in ignore_cols and df[c].dtype == "object" and df[c].str.startswith("[").all()]
+        # convert string representation of list to list
+        # df[list_cols] = df[list_cols].apply(lambda x: eval(x, {'__builtins__': None}) if pd.notna(x) else x)
+        try:
+            df[list_cols] = df[list_cols].fillna('').applymap(eval, na_action='ignore')
+        except ValueError as e:
+            print(f'Error loading {file_path!r} when converting columns {list_cols} to list: {e}', flush=True)
+
+    else:
+        # Download the history from wandb and add metadata
+        run = api.run(run_path)
+        df = pd.DataFrame(list(run.scan_history()))
+
+        # Remove rows with missing completions or rewards, which will be stuff related to weights
+        df.dropna(subset=df.filter(regex='completions|rewards').columns, how='any', inplace=True)
+
+        print(f'Downloaded {df.shape[0]} events from {run_path!r} with id {run_id!r}')
+
+        # Clean and explode dataframe
+        # overwrite object to free memory
+        float_cols = df.filter(regex='reward').columns
+        df = explode_data(clean_data(df)).astype({c: float for c in float_cols}).fillna({c: 0 for c in float_cols})
+
+        if save:
+            df.to_parquet(file_path, index=False)
+
+    # Convert timestamp to datetime.
+    df._timestamp = pd.to_datetime(df._timestamp, unit="s")
+    return df.sort_values("_timestamp")
+
+
+def calculate_stats(df_long, freq='H', save_path=None, ntop=3 ):
+
+    df_long._timestamp = pd.to_datetime(df_long._timestamp)
+
+    # if dataframe has columns such as followup_completions and answer_completions, convert to multiple rows
+    if 'completions' not in df_long.columns:
+        df_long.set_index(['_timestamp','run_id'], inplace=True)
+        df_schema = pd.concat([
+            df_long[['followup_completions','followup_rewards']].rename(columns={'followup_completions':'completions', 'followup_rewards':'rewards'}),
+            df_long[['answer_completions','answer_rewards']].rename(columns={'answer_completions':'completions', 'answer_rewards':'rewards'})
+        ])
+        df_long = df_schema.reset_index()
+
+    run_id = df_long['run_id'].iloc[0]
+    # print(f'Calculating stats for run {run_id!r} dataframe with shape {df_long.shape}')
+
+    # Approximate number of tokens in each completion
+    df_long['completion_num_tokens'] = (df_long['completions'].astype(str).str.split().str.len() / 0.75).round()
+
+    # TODO: use named aggregations
+    reward_aggs = ['sum','mean','std','median','max',aggregate.nonzero_rate, aggregate.nonzero_mean, aggregate.nonzero_std, aggregate.nonzero_median]
+    aggs = {
+        'completions': ['nunique','count', aggregate.diversity, aggregate.successful_diversity, aggregate.success_rate],
+        'completion_num_tokens': ['mean', 'std', 'median', 'max'],
+        **{k: reward_aggs for k in df_long.filter(regex='reward') if df_long[k].nunique() > 1}
+    }
+
+    # Calculate tokens per second
+    if 'completion_times' in df_long.columns:
+        df_long['tokens_per_sec'] = df_long['completion_num_tokens']/(df_long['completion_times']+1e-6)
+        aggs.update({
+            'completion_times': ['mean','std','median','min','max'],
+            'tokens_per_sec': ['mean','std','median','max'],
+        })
+
+    grouper = df_long.groupby(pd.Grouper(key='_timestamp', axis=0, freq=freq))
+    # carry out main aggregations
+    stats = grouper.agg(aggs)
+    # carry out multi-column aggregations using apply
+    diversity = grouper.apply(aggregate.successful_nonzero_diversity)
+    # carry out top completions aggregations using apply
+    top_completions = grouper.apply(aggregate.completion_top_stats, exclude='', ntop=ntop).unstack()
+    
+    # combine all aggregations, which have the same index
+    stats = pd.concat([stats, diversity, top_completions], axis=1)
+    
+    # flatten multiindex columns
+    stats.columns = ['_'.join([str(cc) for cc in c]) if isinstance(c, tuple) else str(c) for c in stats.columns]
+    stats = stats.reset_index().assign(run_id=run_id)
+    
+    if save_path:
+        stats.to_csv(save_path, index=False)
+
+    return stats
+
+
+
+def process(run, load=True, save=False, load_stats=True, freq='H', ntop=3):
+
+    try:
+
+        stats_path = f'data/aggs/stats-{run["run_id"]}.csv'
+        if load_stats and os.path.exists(stats_path):
+            print(f'Loaded stats file {stats_path!r}')
+            return pd.read_csv(stats_path)
+
+        # Load data and add extra columns from wandb run
+        df_long = load_data(run_id=run['run_id'],
+                    run_path=run['run_path'],
+                    load=load,
+                    save=save,
+                    # save = (run['state'] != 'running') & run['end_time']
+                    ).assign(**run.to_dict())
+        assert isinstance(df_long, pd.DataFrame), f'Expected dataframe, but got {type(df_long)}'
+
+        # Get and save stats
+        return calculate_stats(df_long, freq=freq, save_path=stats_path, ntop=ntop)
+
+    except Exception as e:
+        print(f'Error processing run {run["run_id"]!r}:\t{e.__class__.__name__}: {e}',flush=True)
+        print(traceback.format_exc())
+
+def line_chart(df, col, title=None):
+    title = title or col.replace('_',' ').title()
+    fig = px.line(df.astype({'_timestamp':str}),
+            x='_timestamp', y=col,
+            line_group='run_id',
+            title=f'{title} over Time',
+            labels={'_timestamp':'', col: title, 'uids':'UID','value':'counts', 'variable':'Completions'},
+            width=800, height=600,
+            template='plotly_white',
+        ).update_traces(opacity=0.2)
+
+    fig.write_image(f'data/figures/{col}.png')
+    fig.write_html(f'data/figures/{col}.html')
+    return col
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description='Process wandb validator runs for a given netuid.')
+    parser.add_argument('--load_runs',action='store_true', help='Load runs from file.')
+    parser.add_argument('--repull_unfinished',action='store_true', help='Re-pull runs that were running when downloaded and saved.')
+    parser.add_argument('--netuid', type=int, default=None, help='Network UID to use.')
+    parser.add_argument('--ntop', type=int, default=1000, help='Number of runs to process.')
+    parser.add_argument('--min_steps', type=int, default=100, help='Minimum number of steps to include.')
+    parser.add_argument('--max_workers', type=int, default=32, help='Max workers to use.')
+    parser.add_argument('--no_plot',action='store_true', help='Prevent plotting.')    
+    parser.add_argument('--no_save',action='store_true', help='Prevent saving data to file.')
+    parser.add_argument('--no_load',action='store_true', help='Prevent loading downloaded data from file.')
+    parser.add_argument('--no_load_stats',action='store_true', help='Prevent loading stats data from file.')
+    parser.add_argument('--freq', type=str, default='H', help='Frequency to aggregate data.')
+    parser.add_argument('--completions_ntop', type=int, default=3, help='Number of top completions to include in stats.')
+
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+
+    # TODO: flag to overwrite runs that were running when downloaded and saved: check if file date is older than run end time.
+
+    args = parse_arguments()
+    print(args)
+
+    filters = None# {"tags": {"$in": [f'1.1.{i}' for i in range(10)]}}
+    # filters={'tags': {'$in': ['5F4tQyWrhfGVcNhoqeiNsR6KjD4wMZ2kfhLj4oHYuyHbZAc3']}} # Is foundation validator
+    if args.load_runs and os.path.exists('data/wandb.csv'):
+        df_runs = pd.read_csv('data/wandb.csv')
+        assert len(df_runs) >= args.ntop, f'Loaded {len(df_runs)} runs, but expected at least {args.ntop}'
+        df_runs = df_runs.iloc[:args.ntop]
+    else:
+        df_runs = pull_wandb_runs(ntop=args.ntop,
+                                min_steps=args.min_steps,
+                                netuid=args.netuid,
+                                filters=filters
+                                )#summary_filters=lambda s: s.get('augment_prompt'))
+        df_runs.to_csv('data/wandb.csv', index=False)
+
+
+    os.makedirs('data/runs/', exist_ok=True)
+    os.makedirs('data/aggs/', exist_ok=True)
+    os.makedirs('data/figures/', exist_ok=True)
+
+    display(df_runs)
+    if not args.no_plot:
+        plot_gantt(df_runs)
+
+    with ProcessPoolExecutor(max_workers=min(args.max_workers, df_runs.shape[0])) as executor:
+        futures = [executor.submit(
+                            process,
+                            run,
+                            load=not args.no_load,
+                            save=not args.no_save,
+                            load_stats=not args.no_load_stats,
+                            freq=args.freq,
+                            ntop=args.completions_ntop
+                    )
+                   for _, run in df_runs.iterrows()
+                   ]
+
+        # Use tqdm to add a progress bar
+        results = []
+        with tqdm.tqdm(total=len(futures)) as pbar:
+            for future in futures:
+                try:
+                    result = future.result()
+                    results.append(result)
+                except Exception as e:
+                    print(f'-----------------------------\nWorker generated an exception in "process" function:\n{e.__class__.__name__}: {e}\n-----------------------------\n',flush=True)
+                pbar.update(1)
+
+    if not results:
+        raise ValueError('No runs were successfully processed.')
+    print(f'Processed {len(results)} runs.',flush=True)
+
+   # Concatenate the results into a single dataframe
+    df = pd.concat(results, ignore_index=True).sort_values(['_timestamp','run_id'], ignore_index=True)
+
+    df.to_csv('data/processed.csv', index=False)
+    print(f'Saved {df.shape[0]} rows to data/processed.csv')
+
+    display(df)
+    print(f'Unique values in columns:')
+    display(df.nunique().sort_values())
+    if not args.no_plot:
+        
+        plots = []
+
+        cols = df.set_index(['run_id','_timestamp']).columns
+        with ProcessPoolExecutor(max_workers=min(args.max_workers, len(cols))) as executor:
+            futures = [executor.submit(line_chart, df, c) for c in cols]
+
+            # Use tqdm to add a progress bar
+            results = []
+            with tqdm.tqdm(total=len(futures)) as pbar:
+                for future in futures:
+                    try:
+                        result = future.result()
+                        plots.append(result)
+                    except Exception as e:
+                        print(f'-----------------------------\nWorker generated an exception in "line_chart" function:\n{e.__class__.__name__}: {e}\n-----------------------------\n',flush=True)
+                        # traceback.print_exc()                    
+                    pbar.update(1)
+        
+        print(f'Saved {len(plots)} plots to data/figures/')
+
+

opendashboards/assets/inspect.py

@@ -3,6 +3,9 @@ import streamlit as st
 import pandas as pd
 import opendashboards.utils.utils as utils
 
+def clean_data(df):
+    return df.dropna(subset=df.filter(regex='completions|rewards').columns, how='any')
+
 @st.cache_data
 def explode_data(df):
     list_cols = utils.get_list_col_lengths(df)
@@ -19,19 +22,9 @@ def explode_data(df):
 def completions(df_long, col):
     return df_long[col].value_counts()
 
-@st.cache_data
-def weights(df, index='_timestamp'):
-    # Create a column for each UID and show most recent rows
-    scores = df['moving_averaged_scores'].apply(pd.Series).fillna(method='ffill')
-    if index in df.columns:
-        scores.index = df[index]
-
-    # rename columns
-    scores.rename({i: f'UID-{i}' for i in range(scores.shape[1])}, axis=1, inplace=True)
-    return scores    
-    
+
 def run_event_data(df_runs, df, selected_runs):
-    
+
     st.markdown('#')
 
     show_col1, show_col2 = st.columns(2)
@@ -51,4 +44,7 @@ def run_event_data(df_runs, df, selected_runs):
                     column_config={
                         "url": st.column_config.LinkColumn("URL"),
                     }
-        )
+        )
+
+def highlight_row(row, expr, color='lightgrey', bg_color='white'):
+    return [f'background-color:{color}' if expr else f'background-color:{bg_color}'] * len(row)

opendashboards/assets/io.py

@@ -5,19 +5,36 @@ import streamlit as st
 
 import  opendashboards.utils.utils as utils
 
+from pandas.api.types import (
+    is_categorical_dtype,
+    is_datetime64_any_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+)
+
 
 @st.cache_data
 def load_runs(project, filters, min_steps=10):
     runs = []
+    n_events = 0
+    successful = 0
+    progress = st.progress(0, 'Fetching runs from wandb')
     msg = st.empty()
-    for run in utils.get_runs(project, filters, api_key=st.secrets['WANDB_API_KEY']):
-        step = run.summary.get('_step',0)
+
+    all_runs = utils.get_runs(project, filters, api_key=st.secrets['WANDB_API_KEY'])
+    for i, run in enumerate(all_runs):
+        
+        summary = run.summary
+        step = summary.get('_step',-1) + 1
         if step < min_steps:
             msg.warning(f'Skipped run `{run.name}` because it contains {step} events (<{min_steps})')
             continue
-
-        duration = run.summary.get('_runtime')
-        end_time = run.summary.get('_timestamp')
+        
+        prog_msg = f'Loading data {i/len(all_runs)*100:.0f}% ({successful}/{len(all_runs)} runs, {n_events} events)'
+        progress.progress(i/len(all_runs),f'{prog_msg}... **fetching** `{run.name}`')
+        
+        duration = summary.get('_runtime')
+        end_time = summary.get('_timestamp')
         # extract values for selected tags
         rules = {'hotkey': re.compile('^[0-9a-z]{48}$',re.IGNORECASE), 'version': re.compile('^\\d\.\\d+\.\\d+$'), 'spec_version': re.compile('\\d{4}$')}
         tags = {k: tag for k, rule in rules.items() for tag in run.tags if rule.match(tag)}
@@ -27,17 +44,22 @@ def load_runs(project, filters, min_steps=10):
         runs.append({
             'state': run.state,
             'num_steps': step,
+            'num_completions': step*sum(len(v) for k, v in run.summary.items() if k.endswith('completions') and isinstance(v, list)),
             'entity': run.entity,
-            'id': run.id,
-            'name': run.name,
+            'run_id': run.id,
+            'run_name': run.name,
             'project': run.project,
             'url': run.url,
-            'path': os.path.join(run.entity, run.project, run.id),
+            'run_path': os.path.join(run.entity, run.project, run.id),
             'start_time': pd.to_datetime(end_time-duration, unit="s"),
             'end_time': pd.to_datetime(end_time, unit="s"),
-            'duration': pd.to_datetime(duration, unit="s"),
+            'duration': pd.to_timedelta(duration, unit="s").round('s'),
             **tags
         })
+        n_events += step
+        successful += 1
+
+    progress.empty()
     msg.empty()
     return pd.DataFrame(runs).astype({'state': 'category', 'hotkey': 'category', 'version': 'category', 'spec_version': 'category'})
 
@@ -56,7 +78,7 @@ def load_data(selected_runs, load=True, save=False):
         run = selected_runs.loc[idx]
         prog_msg = f'Loading data {i/len(selected_runs)*100:.0f}% ({successful}/{len(selected_runs)} runs, {n_events} events)'
 
-        file_path = os.path.join('data',f'history-{run.id}.csv')
+        file_path = os.path.join('data',f'history-{run.run_id}.csv')
 
         if load and os.path.exists(file_path):
             progress.progress(i/len(selected_runs),f'{prog_msg}... **reading** `{file_path}`')
@@ -67,18 +89,19 @@ def load_data(selected_runs, load=True, save=False):
                 st.exception(e)
                 continue
         else:
-            progress.progress(i/len(selected_runs),f'{prog_msg}... **downloading** `{run.path}`')
+            progress.progress(i/len(selected_runs),f'{prog_msg}... **downloading** `{run.run_path}`')
             try:
-                # Download the history from wandb
-                df = utils.download_data(run.path)
-                # Add metadata to the dataframe
-                df.assign(**run.to_dict())
+                # Download the history from wandb and add metadata
+                df = utils.download_data(run.run_path).assign(**run.to_dict())
 
+                print(f'Downloaded {df.shape[0]} events from `{run.run_path}`. Columns: {df.columns}')
+                df.info()
+                
                 if save and run.state != 'running':
                     df.to_csv(file_path, index=False)
                     # st.info(f'Saved history to {file_path}')
             except Exception as e:
-                info.warning(f'Failed to download history for `{run.path}`')
+                info.warning(f'Failed to download history for `{run.run_path}`')
                 st.exception(e)
                 continue
 
@@ -94,3 +117,92 @@ def load_data(selected_runs, load=True, save=False):
     return pd.concat(frames)
 
 
+def filter_dataframe(df: pd.DataFrame, demo_selection=None) -> pd.DataFrame:
+    """
+    Adds a UI on top of a dataframe to let viewers filter columns
+
+    Args:
+        df (pd.DataFrame): Original dataframe
+        demo_selection (pd.Index): Index of runs to select (if demo)
+
+    Returns:
+        pd.DataFrame: Filtered dataframe
+    """
+    filter_mode = st.sidebar.radio("Filter mode", ("Use demo", "Add filters"), index=0)
+
+    run_msg = st.info("Select a single wandb run or compare multiple runs")
+
+    if filter_mode == "Use demo":
+        df = df.loc[demo_selection]
+        run_msg.info(f"Selected {len(df)} runs")
+        return df
+    
+    df = df.copy()
+
+    # Try to convert datetimes into a standarrd format (datetime, no timezone)
+    for col in df.columns:
+        if is_object_dtype(df[col]):
+            try:
+                df[col] = pd.to_datetime(df[col])
+            except Exception:
+                pass
+
+        if is_datetime64_any_dtype(df[col]):
+            df[col] = df[col].dt.tz_localize(None)
+
+    modification_container = st.container()
+
+    with modification_container:
+        to_filter_columns = st.multiselect("Filter dataframe on", df.columns)
+        for column in to_filter_columns:
+            left, right = st.columns((1, 20))
+            # Treat columns with < 10 unique values as categorical
+            if is_categorical_dtype(df[column]) or df[column].nunique() < 10:
+                user_cat_input = right.multiselect(
+                    f"Values for {column}",
+                    df[column].unique(),
+                    default=list(df[column].unique()),
+                )
+                df = df[df[column].isin(user_cat_input)]
+            elif is_numeric_dtype(df[column]):
+                _min = float(df[column].min())
+                _max = float(df[column].max())
+                step = (_max - _min) / 100
+                user_num_input = right.slider(
+                    f"Values for {column}",
+                    min_value=_min,
+                    max_value=_max,
+                    value=(_min, _max),
+                    step=step,
+                )
+                df = df[df[column].between(*user_num_input)]
+            elif is_datetime64_any_dtype(df[column]):
+                user_date_input = right.date_input(
+                    f"Values for {column}",
+                    value=(
+                        df[column].min(),
+                        df[column].max(),
+                    ),
+                )
+                if len(user_date_input) == 2:
+                    user_date_input = tuple(map(pd.to_datetime, user_date_input))
+                    start_date, end_date = user_date_input
+                    df = df.loc[df[column].between(start_date, end_date)]
+            else:
+                user_text_input = right.text_input(
+                    f"Substring or regex in {column}",
+                )
+                if user_text_input:
+                    df = df[df[column].astype(str).str.contains(user_text_input)]
+
+
+    # Load data if new runs selected
+    if len(df):
+        run_msg.info(f"Selected {len(df)} runs")
+    else:
+        # open a dialog to select runs
+        run_msg.error("Please select at least one run")
+        # st.snow()
+        # st.stop()
+
+    return df

opendashboards/assets/metric.py

@@ -2,6 +2,18 @@ import time
 import pandas as pd
 import streamlit as st
 
+def fmt(number):
+    units = ['', 'k', 'M', 'B']
+    magnitude = 0
+    while abs(number) >= 1000 and magnitude < len(units) - 1:
+        magnitude += 1
+        number /= 1000
+
+    if units[magnitude]:
+        return f'{number:.2f}{units[magnitude]}'
+    else:
+        return f'{number:.0f}{units[magnitude]}'
+
 
 @st.cache_data
 def wandb(df_runs):
@@ -9,50 +21,72 @@ def wandb(df_runs):
     # get rows where start time is older than 24h ago
     df_runs_old = df_runs.loc[df_runs.start_time < pd.to_datetime(time.time()-24*60*60, unit='s')]
 
-    col1, col2, col3 = st.columns(3)
+    col1, col2, col3, col4 = st.columns(4)
 
-    col1.metric('Runs', df_runs.shape[0], delta=f'{df_runs.shape[0]-df_runs_old.shape[0]} (24h)')
-    col2.metric('Hotkeys', df_runs.hotkey.nunique(), delta=f'{df_runs.hotkey.nunique()-df_runs_old.hotkey.nunique()} (24h)')
-    col3.metric('Events', df_runs.num_steps.sum(), delta=f'{df_runs.num_steps.sum()-df_runs_old.num_steps.sum()} (24h)')
+    # Convert to appropriate units e.g. 1.2k instead of 1200.
+    col1.metric('Runs', fmt(df_runs.shape[0]), delta=fmt(df_runs.shape[0]-df_runs_old.shape[0])+' (24h)')
+    col2.metric('Hotkeys', fmt(df_runs.hotkey.nunique()), delta=fmt(df_runs.hotkey.nunique()-df_runs_old.hotkey.nunique())+' (24h)')
+    col3.metric('Events', fmt(df_runs.num_steps.sum()), delta=fmt(df_runs.num_steps.sum()-df_runs_old.num_steps.sum())+' (24h)')
+    col4.metric('Completions', fmt(df_runs.num_completions.sum()), delta=fmt(df_runs.num_completions.sum()-df_runs_old.num_completions.sum())+' (24h)')
+    
     st.markdown('----')
 
 
 @st.cache_data
-def runs(df, df_long, selected_runs):
-
-    col1, col2, col3 = st.columns(3)
-    col1.metric(label="Runs", value=len(selected_runs))
-    col1.metric(label="Events", value=df.shape[0]) #
-    col2.metric(label="Followup UIDs", value=df_long.followup_uids.nunique())
-    col2.metric(label="Answer UIDs", value=df_long.answer_uids.nunique())
-    col3.metric(label="Followup Completions", value=df_long.followup_completions.nunique())
-    col3.metric(label="Answer Completions", value=df_long.answer_completions.nunique())
+def runs(df_long):
+    
+    col1, col2, col3, col4 = st.columns(4)
+    print(df_long.columns)
+
+    # Convert to appropriate units e.g. 1.2k instead of 1200.c
+    col1.metric('Runs', fmt(df_long.run_id.nunique()))
+    col2.metric('Hotkeys', fmt(df_long.hotkey.nunique()))
+    col3.metric('Events', fmt(df_long.groupby(['run_id','_step']).ngroups))
+    col4.metric('Completions', fmt(df_long.shape[0]))
+    
+    name_type = df_long.name.apply(lambda x: x if not x[-1].isdigit() else x[:-1])
+    aggs = df_long.groupby(name_type).agg({'uids': 'nunique', 'completions': 'nunique'})
+    print(aggs)
+    for i,c in enumerate(st.columns(len(aggs))):
+        name = aggs.index[i].title()
+        uid_unique, comp_unique = aggs.iloc[i]
+        c.metric(label=f'{name} UIDs', value=uid_unique) 
+        c.metric(label=f'{name} Completions', value=comp_unique)
+
     st.markdown('----')
 
 
-    
+
 @st.cache_data
-def uids(df_long, src, uid=None):
+def uids(df_long, src, uids=None):
 
-    uid_col = f'{src}_uids'
-    completion_col = f'{src}_completions'
     nsfw_col = f'{src}_nsfw_scores'
-    reward_col = f'{src}_rewards'
 
-    if uid is not None:
-        df_long = df_long.loc[df_long[uid_col] == uid]
+    if uids:
+        df_long = df_long.loc[df_long['uids'].isin(uids)]
 
-    col1, col2, col3 = st.columns(3)
+    col1, col2, col3, col4 = st.columns(4)
     col1.metric(
         label="Success %",
-        value=f'{df_long.loc[df_long[completion_col].str.len() > 0].shape[0]/df_long.shape[0] * 100:.1f}'
+        value=f'{df_long.loc[df_long["completions"].str.len() > 0].shape[0]/df_long.shape[0] * 100:.1f}',
+        help='Number of successful completions divided by total number of events'
     )
     col2.metric(
         label="Diversity %",
-        value=f'{df_long[completion_col].nunique()/df_long.shape[0] * 100:.1f}'
+        value=f'{df_long["completions"].nunique()/df_long.shape[0] * 100:.1f}',
+        help='Number of unique completions divided by total number of events'
     )
+    # uniqueness can be expressed as the average number of unique completions per uid divided by all unique completions
+    # uniqueness is the shared completions between selected uids 
+
     col3.metric(
+        label="Uniqueness %",
+        value=f'{df_long.groupby("uids")["completions"].nunique().mean()/df_long["completions"].nunique() * 100:.1f}',
+        help='Average number of unique completions per uid divided by all unique completions'
+    )
+    col4.metric(
         label="Toxicity %",
-        value=f'{df_long[nsfw_col].mean() * 100:.1f}' if nsfw_col in df_long.columns else 'N/A'
+        value=f'{df_long[nsfw_col].mean() * 100:.1f}' if nsfw_col in df_long.columns else '--',
+        help='Average toxicity score of all events'
     )
     st.markdown('----')

opendashboards/assets/plot.py

@@ -8,6 +8,8 @@ def uid_diversty(df, rm_failed=True):
         plotting.plot_uid_diversty(
             df,
             remove_unsuccessful=rm_failed
+        ).update_layout(
+            coloraxis_showscale=False,
         ),
         use_container_width=True
     )
@@ -22,6 +24,8 @@ def leaderboard(df, ntop, group_on, agg_col, agg, alias=False):
             agg_col=agg_col,
             agg=agg,
             alias=alias
+        ).update_layout(
+            coloraxis_showscale=False,
         ),
         use_container_width=True
     )
@@ -49,4 +53,30 @@ def weights(df, uids, ntop=10):
             ntop=ntop
         ),
         use_container_width=True
+    )
+
+def completion_length_time(df, completion_col, uid_col, time_col, length_opt='characters'):
+    return st.plotly_chart(
+        plotting.plot_completion_length_time(
+            df,
+            uid_col=uid_col,
+            completion_col=completion_col,
+            time_col=time_col,
+            length_opt=length_opt
+        ),
+        use_container_width=True
+    )
+
+def uid_completion_counts(df, uids, src, rm_empty, ntop=100, cumulative=False, normalize=True):
+    return st.plotly_chart(
+        plotting.plot_uid_completion_counts(
+            df,
+            uids=uids,
+            src=src,
+            rm_empty=rm_empty,
+            ntop=ntop,
+            cumulative=cumulative,
+            normalize=normalize
+        ),
+        use_container_width=True
     )

opendashboards/utils/aggregate.py

@@ -0,0 +1,52 @@
+import pandas as pd
+
+def diversity(x):
+    return x.nunique()/len(x) if len(x)>0 else 0
+
+def _nonempty(x):
+    return x[x.astype(str).str.len()>0]
+
+def successful_diversity(x):
+    return diversity(_nonempty(x))
+
+def success_rate(x):
+    return len(_nonempty(x))/len(x) if len(x)>0 else 0
+
+def threshold_rate(x, threshold):
+    return (x>threshold).sum()/len(x)
+
+def successful_nonzero_diversity(x):
+    # To be used with groupby.apply
+    return pd.Series({'completions_successful_nonzero_diversity': successful_diversity(x.loc[x['rewards']>0,'completions'])})
+    
+def completion_top_stats(x, exclude=None, ntop=1):
+    # To be used with groupby.apply
+    vc = x['completions'].value_counts()
+    if exclude is not None:
+        vc.drop(exclude, inplace=True, errors='ignore')
+        
+    rewards = x.loc[x['completions'].isin(vc.index[:ntop])].groupby('completions').rewards.agg(['mean','std','max'])
+    return pd.DataFrame({
+        'completions_top':rewards.index.tolist(),
+        'completions_freq':vc.values[:ntop],
+        'completions_reward_mean':rewards['mean'].values,
+        'completions_reward_std':rewards['std'].values
+        })
+
+def top(x, i=0, exclude=''):
+    return _nonempty(x).value_counts().drop(exclude, errors='ignore').index[i]
+
+def freq(x, i=0, exclude=''):
+    return _nonempty(x).value_counts().drop(exclude, errors='ignore').values[i]
+
+def nonzero_rate(x):
+    return (x>0).sum()/len(x)
+
+def nonzero_mean(x):
+    return x[x>0].mean()
+
+def nonzero_std(x):
+    return x[x>0].std()
+
+def nonzero_median(x):
+    return x[x>0].median()

opendashboards/utils/plotting.py

@@ -44,7 +44,7 @@ def plot_throughput(df: pd.DataFrame, n_minutes: int = 10) -> go.Figure:
 
 
 def plot_weights(scores: pd.DataFrame, ntop: int = 20, uids: List[Union[str, int]] = None) -> go.Figure:
-    """_summary_
+    """Plot weights of uids.
 
     Args:
         scores (pd.DataFrame): Dataframe of scores. Should be indexed by timestamp and have one column per uid.
@@ -62,16 +62,16 @@ def plot_weights(scores: pd.DataFrame, ntop: int = 20, uids: List[Union[str, int
     ).update_traces(opacity=0.7)
 
 
-def plot_uid_diversty(df: pd.DataFrame, remove_unsuccessful: bool = False) -> go.Figure:
+def plot_uid_diversty(df: pd.DataFrame, x: str = 'followup', y: str = 'answer', remove_unsuccessful: bool = False) -> go.Figure:
     """Plot uid diversity as measured by ratio of unique to total completions.
 
     Args:
         df (pd.DataFrame): Dataframe of event log.
     """
-    uid_cols = ["followup_uids", "answer_uids"]
-    completion_cols = ["followup_completions", "answer_completions"]
-    reward_cols = ["followup_rewards", "answer_rewards"]
-    list_cols = uid_cols + completion_cols + reward_cols
+    return px.scatter(x=[1,2,3],y=[1,2,3])
+    xrows = df.loc[df.name.str.contains(x)]
+    yrows = df.loc[df.name.str.contains(y)]
+    df = pd.merge(xrows, yrows, on='uid', suffixes=('_followup', '_answer'))
 
     df = df[list_cols].explode(column=list_cols)
     if remove_unsuccessful:
@@ -88,7 +88,7 @@ def plot_uid_diversty(df: pd.DataFrame, remove_unsuccessful: bool = False) -> go
         frames.append(frame)
 
     merged = pd.merge(*frames, left_index=True, right_index=True, suffixes=("_followup", "_answer"))
-    merged["reward_mean"] = merged.filter(regex="rewards_mean").mean(axis=1)
+    merged["reward_mean"] = merged.filter(regex="rewards_mean").mean(axis=1).astype(float)
 
     merged.index.name = "UID"
     merged.reset_index(inplace=True)
@@ -97,8 +97,8 @@ def plot_uid_diversty(df: pd.DataFrame, remove_unsuccessful: bool = False) -> go
         merged,
         x="diversity_followup",
         y="diversity_answer",
-        opacity=0.3,
-        size="followup_completions_size",
+        opacity=0.35,
+        # size="completions_size",
         color="reward_mean",
         hover_data=["UID"] + merged.columns.tolist(),
         marginal_x="histogram",
@@ -112,7 +112,7 @@ def plot_uid_diversty(df: pd.DataFrame, remove_unsuccessful: bool = False) -> go
 
 def plot_completion_rates(
     df: pd.DataFrame,
-    msg_col: str = "all_completions",
+    msg_col: str = "completions",
     time_interval: str = "H",
     time_col: str = "_timestamp",
     ntop: int = 20,
@@ -123,7 +123,7 @@ def plot_completion_rates(
 
     Args:
         df (pd.DataFrame): Dataframe of event log.
-        msg_col (str, optional): List-like column containing completions. Defaults to 'all_completions'.
+        msg_col (str, optional): List-like column containing completions. Defaults to 'completions'.
         time_interval (str, optional): Pandas time interval. Defaults to 'H'. See https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#timeseries-offset-aliases
         time_col (str, optional): Column containing timestamps as pd.Datetime. Defaults to '_timestamp'.
         ntop (int, optional): Number of completions to plot. Defaults to 20.
@@ -163,10 +163,10 @@ def plot_completion_rates(
 
 def plot_completion_rewards(
     df: pd.DataFrame,
-    msg_col: str = "followup_completions",
-    reward_col: str = "followup_rewards",
+    msg_col: str = "completions",
+    reward_col: str = "rewards",
     time_col: str = "_timestamp",
-    uid_col: str = "followup_uids",
+    uid_col: str = "uids",
     ntop: int = 3,
     completions: List[str] = None,
     completion_regex: str = None,
@@ -175,9 +175,10 @@ def plot_completion_rewards(
 
     Args:
         df (pd.DataFrame): Dataframe of event log.
-        msg_col (str, optional): List-like column containing completions. Defaults to 'followup_completions'.
-        reward_col (str, optional): List-like column containing rewards. Defaults to 'followup_rewards'.
+        msg_col (str, optional): List-like column containing completions. Defaults to 'completions'.
+        reward_col (str, optional): List-like column containing rewards. Defaults to 'rewards'.
         time_col (str, optional): Column containing timestamps as pd.Datetime. Defaults to '_timestamp'.
+        uid_col (str, optional): Column containing UIDs. Defaults to 'uids'.
         ntop (int, optional): Number of completions to plot. Defaults to 20.
         completions (List[str], optional): List of completions to plot. Defaults to None.
         completion_regex (str, optional): Regex to match completions. Defaults to None.
@@ -198,7 +199,11 @@ def plot_completion_rewards(
         else:
             completions = completion_counts.index[:ntop]
             print(f"Using top {len(completions)} completions: \n{completions}")
-
+    else:
+        found_completions = [c for c in completions if c in completion_counts.index]
+        print(f"Using {len(found_completions)}/{len(completions)} completions: \n{found_completions}")
+        completions = found_completions
+        
     # Get ranks of completions in terms of number of occurrences
     ranks = completion_counts.rank(method="dense", ascending=False).loc[completions].astype(int)
 
@@ -219,14 +224,14 @@ def plot_completion_rewards(
         labels={"rank": "Rank", reward_col: "Reward", time_col: ""},
         title=f"Rewards for {len(completions)} Messages",
         **plotly_config,
-        opacity=0.3,
+        opacity=0.35,
     )
 
 
 def plot_leaderboard(
     df: pd.DataFrame,
-    group_on: str = "answer_uids",
-    agg_col: str = "answer_rewards",
+    group_on: str = "uids",
+    agg_col: str = "rewards",
     agg: str = "mean",
     ntop: int = 10,
     alias: bool = False,
@@ -235,44 +240,44 @@ def plot_leaderboard(
 
     Args:
         df (pd.DataFrame): Dataframe of event log.
-        group_on (str, optional): Entities to use for grouping. Defaults to 'answer_uids'.
-        agg_col (str, optional): Column to aggregate. Defaults to 'answer_rewards'.
+        group_on (str, optional): Entities to use for grouping. Defaults to 'uids'.
+        agg_col (str, optional): Column to aggregate. Defaults to 'rewards'.
         agg (str, optional): Aggregation function. Defaults to 'mean'.
         ntop (int, optional): Number of entities to plot. Defaults to 10.
         alias (bool, optional): Whether to use aliases for indices. Defaults to False.
     """
     df = df[[group_on, agg_col]].explode(column=[group_on, agg_col])
 
-    rankings = df.groupby(group_on)[agg_col].agg(agg).sort_values(ascending=False).head(ntop)
+    rankings = df.groupby(group_on)[agg_col].agg(agg).sort_values(ascending=False).head(ntop).astype(float)
     if alias:
         index = rankings.index.map({name: str(i) for i, name in enumerate(rankings.index)})
     else:
         index = rankings.index.astype(str)
 
-    print(f"Using top {ntop} {group_on} by {agg_col}: \n{rankings}")
     return px.bar(
-        x=rankings.astype(float),
+        x=rankings,
         y=index,
         color=rankings,
         orientation="h",
         labels={"x": f"{agg_col.title()}", "y": group_on, "color": ""},
         title=f"Leaderboard for {agg_col}, top {ntop} {group_on}",
         color_continuous_scale="BlueRed",
-        opacity=0.5,
+        opacity=0.35,
         hover_data=[rankings.index.astype(str)],
         **plotly_config,
     )
 
 
+
 def plot_dendrite_rates(
-    df: pd.DataFrame, uid_col: str = "answer_uids", reward_col: str = "answer_rewards", ntop: int = 20, uids: List[int] = None
+    df: pd.DataFrame, uid_col: str = "uids", reward_col: str = "rewards", ntop: int = 20, uids: List[int] = None
 ) -> go.Figure:
     """Makes a bar chart of the success rate of dendrite calls for a given set of uids.
 
     Args:
         df (pd.DataFrame): Dataframe of event log.
-        uid_col (str, optional): Column containing uids. Defaults to 'answer_uids'.
-        reward_col (str, optional): Column containing rewards. Defaults to 'answer_rewards'.
+        uid_col (str, optional): Column containing uids. Defaults to 'uids'.
+        reward_col (str, optional): Column containing rewards. Defaults to 'rewards'.
         ntop (int, optional): Number of uids to plot. Defaults to 20.
         uids (List[int], optional): List of uids to plot. Defaults to None.
 
@@ -297,15 +302,91 @@ def plot_dendrite_rates(
         barmode="group",
         title="Dendrite Calls by UID",
         color_continuous_scale="Blues",
-        opacity=0.5,
+        opacity=0.35,
         **plotly_config,
     )
 
+def plot_completion_length_time(
+    df: pd.DataFrame,
+    uid_col: str = "uids",
+    completion_col: str = "completions",
+    time_col: str = "completion_times",
+    uids: List[int] = None,
+    length_opt: str = 'characters',
+) -> go.Figure:
+
+
+    df = df[[uid_col, completion_col, time_col]].explode(column=[uid_col, completion_col, time_col])
+    df["time"] = df[time_col].astype(float)
+    if uids is not None:
+        df = df.loc[df[uid_col].isin(uids)]
+        
+        
+    if length_opt == 'characters':
+        df["completion_length"] = df[completion_col].str.len()
+    elif length_opt == 'words':
+        df["completion_length"] = df[completion_col].str.split().str.len()
+    elif length_opt == 'sentences':
+        df["completion_length"] = df[completion_col].str.split('.').str.len()
+    else:
+        raise ValueError(f"length_opt must be one of 'words', 'characters', or 'sentences', got {length_opt}")
+
+    return px.scatter(
+        df,
+        x='completion_length',
+        y='time',
+        color=uid_col if uids is not None else None,
+        labels={"completion_length": f"Completion Length, {length_opt.title()}", "time": "Time (s)"},
+        title=f"Completion Length vs Time, {length_opt.title()}",
+        marginal_x="histogram",
+        marginal_y="histogram",
+        hover_data=[uid_col, completion_col],
+        opacity=0.35,
+        **plotly_config,
+    )
+
+def plot_uid_completion_counts(
+    df: pd.DataFrame,
+    uids: List[int],
+    src: str = 'answer',
+    rm_empty: bool = True,
+    ntop: int = 100,
+    cumulative: bool = False,
+    normalize: bool = True,
+) -> go.Figure:
+
+    completion_col = f'completions'
+    uid_col = f'uids'
+    if rm_empty:
+        df = df.loc[df[completion_col].str.len()>0]
+
+    df = df.loc[df[uid_col].isin(uids)]
+
+    g = df.groupby(uid_col)[completion_col].value_counts(normalize=normalize).reset_index(level=1)
+    y_col = g.columns[-1]
+
+    # rescale each group to have a max of 1 if normalize is True
+    if cumulative:
+        g[y_col] = g.groupby(level=0)[y_col].cumsum().transform(lambda x: x/x.max() if normalize else x)
+
+    # get top n completions
+    g = g.groupby(level=0).head(ntop)
+
+    # # create a rank column which increments by one and resets when the uid changes
+    g['rank'] = g.groupby(level=0).cumcount()+1
+
+    return px.line(g.sort_index().reset_index(),
+            x='rank',y=y_col,color=uid_col,
+            labels={'rank':'Top Completions',uid_col:'UID',y_col:y_col.replace('_',' ').title()},
+            title=f'{src.title()} Completion {y_col.replace("_"," ").title()}s by Rank',
+            **plotly_config,
+            ).update_traces(opacity=0.7)
+
 
 def plot_network_embedding(
     df: pd.DataFrame,
-    uid_col: str = "followup_uids",
-    completion_col: str = "followup_completions",
+    uid_col: str = "uids",
+    completion_col: str = "completions",
     ntop: int = 1,
     uids: List[int] = None,
 ) -> go.Figure:
@@ -314,8 +395,8 @@ def plot_network_embedding(
     Args:
         df (pd.DataFrame): Dataframe of event log.
 
-        uid_col (str, optional): Column containing uids. Defaults to 'answer_uids'.
-        completion_col (str, optional): Column containing completions. Defaults to 'followup_completions'.
+        uid_col (str, optional): Column containing uids. Defaults to 'uids'.
+        completion_col (str, optional): Column containing completions. Defaults to 'completions'.
         ntop (int, optional): Number of uids to plot. Defaults to 20.
         hover_data (List[str], optional): Columns to include in hover data. Defaults to None.
         uids (List[int], optional): List of uids to plot. Defaults to None.
@@ -358,6 +439,6 @@ def plot_network_embedding(
         title=f"Graph for Top {ntop} Completion Similarities",
         color_continuous_scale="BlueRed",
         hover_data=["UID", "top_completions"],
-        opacity=0.5,
+        opacity=0.35,
         **plotly_config,
     )