Working metagraph dashboard

meta_plotting.py

@@ -1,31 +1,41 @@
 import numpy as np
+import pandas as pd
 import plotly.express as px
+from plotly.subplots import make_subplots
 
-def plot_trace(df, col='emission', agg='mean', time_col='timestamp', ntop=10, hotkeys=None, hotkey_regex=None, abbrev=8, type='Miners'):
+plotly_config = dict(width=800, height=600, template='plotly_white')
+
+def plot_trace(df, col='emission', agg='mean', time_col='timestamp', ntop=10, hotkeys=None, hotkey_regex=None, abbrev=8, type='Miners', smooth=1, smooth_agg='mean', opacity=0.):
     
     if hotkeys is not None:
         df = df.loc[df.hotkey.isin(hotkeys)]
     if hotkey_regex is not None:
         df = df.loc[df.hotkey.str.contains(hotkey_regex)]
-        
-    top_miners = df.groupby('hotkey')[col].agg(agg).sort_values(ascending=False)
     
+    # select hotkeys with highest average value of col (e.g. emission) over time
+    top_miners = df.groupby('hotkey')[col].agg(agg).sort_values(ascending=False)
+    print(f'Top miners by {col!r}:\n{top_miners}')
     stats = df.loc[df.hotkey.isin(top_miners.index[:ntop])].sort_values(by=time_col)
+
+    # smooth values of col (e.g. emission) over time
+    # stats[col] = stats.groupby('hotkey')[col].rolling(smooth).agg(smooth_agg).values
     
     stats['hotkey_abbrev'] = stats.hotkey.str[:abbrev]
     stats['coldkey_abbrev'] = stats.coldkey.str[:abbrev]
     stats['rank'] = stats.hotkey.map({k:i for i,k in enumerate(top_miners.index, start=1)})
+    print(stats)
     
+    y_label = col.title().replace('_',' ') + f' ({agg})'
     return px.line(stats.sort_values(by=[time_col,'rank']),
-                    x=time_col, y=col, color='coldkey_abbrev', line_group='hotkey_abbrev',
-                    hover_data=['hotkey','rank'],
-                    labels={col:col.title(),'timestamp':'','coldkey_abbrev':f'Coldkey (first {abbrev} chars)','hotkey_abbrev':f'Hotkey (first {abbrev} chars)'},
-                    title=f'Top {ntop} {type}, by {col.title()}',
-                    template='plotly_white', width=800, height=600,
-                    ).update_traces(opacity=0.7)
+                x=time_col, y=col, color='coldkey_abbrev', line_group='hotkey_abbrev',
+                hover_data=['hotkey','rank'],
+                labels={col:y_label,'timestamp':'','coldkey_abbrev':f'Coldkey (first {abbrev} chars)','hotkey_abbrev':f'Hotkey (first {abbrev} chars)'},
+                title=f'Top {ntop} {type}, by {y_label}',
+                **plotly_config
+            ).update_traces(opacity=opacity)
     
     
-def plot_cabals(df, sel_col='coldkey', count_col='hotkey', time_col='timestamp', values=None, ntop=10, abbr=8):
+def plot_cabals(df, sel_col='coldkey', count_col='hotkey', time_col='timestamp', values=None, ntop=10, abbr=8, smooth=1, smooth_agg='mean', opacity=0.):
     
     if values is None:
         values = df[sel_col].value_counts().sort_values(ascending=False).index[:ntop].tolist()
@@ -33,16 +43,106 @@ def plot_cabals(df, sel_col='coldkey', count_col='hotkey', time_col='timestamp',
         
     df = df.loc[df[sel_col].isin(values)]
     rates = df.groupby([time_col,sel_col])[count_col].nunique().reset_index()
+    
+    # smoothing is hard
+    # rates = rates.groupby(level=1).rolling(smooth, min_periods=1).agg(smooth_agg)
+    
     abbr_col = f'{sel_col} (first {abbr} chars)'
     rates[abbr_col] = rates[sel_col].str[:abbr]
     return px.line(rates.melt(id_vars=[time_col,sel_col,abbr_col]), 
-            x=time_col, y='value', color=abbr_col,
-            #facet_col='variable',  facet_col_wrap=1,
-            labels={'value':f'Number of Unique {count_col.title()}s per {sel_col.title()}','timestamp':''}, 
-            category_orders={abbr_col:[ v[:abbr] for v in values]},
-            # title=f'Unique {count_col.title()}s Associated with Selected {sel_col.title()}s in Metagraph',
-            title=f'Impact of Validators Update on Cabal',           
-            width=800, height=600, template='plotly_white',
-            )
+                x=time_col, y='value', color=abbr_col,
+                labels={'value':f'Number of Unique {count_col.title()}s per {sel_col.title()}','timestamp':''}, 
+                category_orders={abbr_col:[ v[:abbr] for v in values]},
+                title=f'Unique {count_col.title()}s Associated with Top {ntop} {sel_col.title()}s',            
+                **plotly_config
+            ).update_traces(opacity=opacity)        
         
-        
+def plot_churn(df, time_col='timestamp', type='changed', step=1, smooth=1, smooth_agg='mean', opacity=0.5):
+    """
+    Produces a plotly figure which shows number of changed hotkeys in each step
+    """
+    
+    def churn(s):
+        results = [{'delta':np.nan}]
+        for i, idx in enumerate(s.index[1:]):
+    
+            curr = s.loc[idx]
+            prev = s.iloc[i]
+            if type == 'changed':
+                delta = curr.symmetric_difference(prev)
+            elif type == 'added':
+                delta = curr.difference(prev)
+            elif type == 'removed':
+                delta = prev.difference(curr)
+            else:
+                raise ValueError(f'Unknown type {type!r}')
+            
+            results.append({'delta': len(delta)})
+    
+        return pd.DataFrame(results, index=s.index)
+ 
+    churn_frame = churn(df.iloc[::step].groupby(['block','timestamp']).hotkey.unique().apply(set))
+    
+    return px.line(churn_frame.rolling(smooth, min_periods=1).agg(smooth_agg).reset_index(),
+                   x=time_col, y='delta', 
+                   labels={'delta':f'Number of {type.title()} Hotkeys','timestamp':''}, 
+                   hover_name='block',
+                   **plotly_config
+            ).update_traces(opacity=opacity)    
+    
+    
+def plot_occupancy(df, time_col='timestamp',step=1, smooth=1, smooth_agg='mean', opacity=0.5):
+    """
+    Produces a plotly figure which shows number of unique hotkeys in each step
+    """
+    occupancy_frame = df.iloc[::step].assign(
+        Type=df.iloc[::step].validator_trust.apply(lambda x: 'Miner' if x==0 else 'Validator')
+        ).groupby(['Type','timestamp','block']).hotkey.nunique()
+
+    # make two plots, with a secondary y axis
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    trace1 = px.line(occupancy_frame.loc['Miner'].rolling(smooth, min_periods=1).agg(smooth_agg).reset_index(), 
+                     x='timestamp',y='hotkey', hover_name='block')
+    trace2 = px.line(occupancy_frame.loc['Validator'].rolling(smooth, min_periods=1).agg(smooth_agg).reset_index(), 
+                     x='timestamp', y='hotkey', hover_name='block')
+    
+    fig.add_trace(trace1.data[0])
+    fig.add_trace(trace2.update_traces(line_color='red').data[0], secondary_y=True, row=1,col=1)
+    
+    fig.update_yaxes(title_text='Miner Hotkeys', secondary_y=False)  # Customize primary y-axis title
+    fig.update_yaxes(title_text='Validator Hotkeys', secondary_y=True, tickfont=dict(color='red'), title=dict(font_color='red'))  # Customize secondary y-axis title
+    
+    return fig.update_layout( **plotly_config).update_traces(opacity=opacity)
+
+def plot_animation(df, x='emission_sum', y='total_stake_sum', color='emission_mean', size='hotkey_nunique', step=10, opacity=0.5):
+    
+    agg_dict = {}
+    for column_name in [x, y, color, size]:
+        column, agg_name = column_name.rsplit('_', 1)
+
+        if column not in agg_dict:
+            agg_dict[column] = [agg_name]
+        else:
+            agg_dict[column].append(agg_name)
+    
+    # select every nth block
+    if step>1:
+        blocks_subset = df.block.unique()[::step]
+        df = df.loc[df.block.isin(blocks_subset)]
+        
+    df_agg = df.groupby(['block','timestamp','coldkey']).agg({'hotkey':'nunique', 'ip':'nunique', **agg_dict})
+    df_agg.columns = ['_'.join(col).strip() for col in df_agg.columns]
+    print(df_agg.columns)
+    
+    return px.scatter(df_agg.reset_index(), 
+                x=x, range_x=[-df_agg[x].max()*0.1, df_agg[x].max()*1.1],
+                y=y, range_y=[-df_agg[y].max()*0.1, df_agg[y].max()*1.1],
+                size=size, 
+                opacity=opacity,
+                color=color, 
+                color_continuous_scale='BlueRed',
+                animation_frame='block', 
+                animation_group='coldkey',
+                hover_name='coldkey',
+                **plotly_config
+            )   

meta_utils.py

@@ -1,4 +1,5 @@
 import os
+import re
 import glob
 import tqdm
 import dill as pickle
@@ -11,10 +12,37 @@ block_time_500k = datetime.datetime(2023, 5, 29, 5, 29, 0)
 block_time_800k = datetime.datetime(2023, 7, 9, 21, 32, 48)
 dt = (pd.Timestamp(block_time_800k)-pd.Timestamp(block_time_500k))/(800_000-500_000)
 
-def run_subprocess(*args):
-    # Trigger the multigraph.py script to run and save metagraph snapshots
-    return subprocess.run('python multigraph.py'.split()+list(args), 
-                          shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)    
+def run_subprocess(command='python multigraph.py', *args):
+    try:
+        # Run the subprocess with stdout and stderr pipes connected
+        command = command + " ".join(args)
+        print(f'{"===="*20}\nRunning: {command!r}')
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,  # Set to True for text mode
+            bufsize=1,  # Line buffered, so output is available line by line
+            shell=True  # Set to True to allow running shell commands (use with caution)
+        )
+
+        print(f'Subprocess started with pid {process.pid} and streaming output from stdout:')
+        with process.stdout as output:
+            for line in output:
+                print(line, end='', flush=True)  # Print without adding an extra newline                
+                if match := re.search('(?P<done>\\d+)/(?P<total>\\d+)',line):
+                    print('---> match.groupdict():', match.groupdict())
+                    # try yielding the line here
+                
+        # Wait for the subprocess to finish and get the return code
+        process.wait()
+
+        print("===="*20)
+        return process.returncode 
+
+    except subprocess.CalledProcessError as e:
+        # If the subprocess returns a non-zero exit code, this exception will be raised
+        return e.returncode
 
 def load_metagraph(path, extra_cols=None, rm_cols=None):
     
@@ -24,7 +52,7 @@ def load_metagraph(path, extra_cols=None, rm_cols=None):
     df = pd.DataFrame(metagraph.axons)
     df['block'] = metagraph.block.item()
     df['timestamp'] = block_time_500k + dt*(df['block']-500_000)
-    df['difficulty'] = metagraph.difficulty
+    df['difficulty'] = getattr(metagraph, 'difficulty', None)
     for c in extra_cols:
         vals = getattr(metagraph,c)
         df[c] = vals        

metadash.py

@@ -1,9 +1,10 @@
 import os
+import time
 import pandas as pd
 import streamlit as st
 from meta_utils import run_subprocess, load_metagraphs
 # from opendashboards.assets import io, inspect, metric, plot
-from meta_plotting import plot_trace, plot_cabals
+import meta_plotting as plotting 
 import asyncio
 from functools import lru_cache
 
@@ -22,8 +23,9 @@ loop = asyncio.new_event_loop()
 asyncio.set_event_loop(loop)
 import bittensor
 
-datadir='data/metagraph/1/'
-blockfiles = sorted(int(filename.split('.')[0]) for filename in os.listdir(datadir))
+netuid = 1
+datadir=f'data/metagraph/{netuid}/'
+blockfiles = sorted(int(filename.split('.')[0]) for filename in os.listdir(datadir) if filename.split('.')[0].isdigit())
 DEFAULT_SRC = 'miner'
 DEFAULT_BLOCK_START = blockfiles[0]
 DEFAULT_BLOCK_END = blockfiles[-1]
@@ -48,27 +50,29 @@ st.title('Metagraph :red[Analysis] Dashboard :eyes:')
 st.markdown('#')
 st.markdown('#')
 
-netuid = 1
 subtensor = bittensor.subtensor(network='finney')
 current_block = subtensor.get_current_block()
 
-@lru_cache(maxsize=1)
-def _metagraph(block):
+@st.cache_data
+def _metagraph(block, subnet):
     
-    print('rerunning cache')
+    print(f'rerunning cache with block {block}')
     return ( 
-            subtensor.metagraph(netuid, block=block), 
-            subtensor.metagraph(netuid, block=block - 7200),
-            subtensor.burn(netuid=netuid, block=block),
-            subtensor.burn(netuid=netuid, block=block - 7200),
+            subtensor.metagraph(subnet, block=block), 
+            subtensor.metagraph(subnet, block=block - 7200),
+            subtensor.burn(netuid=subnet, block=block),
+            subtensor.burn(netuid=subnet, block=block - 7200),
     )
-    
-current_metagraph, yesterday_metagraph, current_burn, yesterday_burn = _metagraph(10*current_block//10)
 
-current_vcount = current_metagraph.validator_permit.sum().item()
-current_mcount = (~current_metagraph.validator_permit).sum().item()
-yesterday_vcount = yesterday_metagraph.validator_permit.sum().item()
-yesterday_mcount = (~yesterday_metagraph.validator_permit).sum().item()
+current_metagraph, yesterday_metagraph, current_burn, yesterday_burn = _metagraph(10*(current_block//10), netuid)
+current_validators = current_metagraph.validator_permit[current_metagraph.validator_trust > 0.0]
+yesterday_validators = yesterday_metagraph.validator_permit[yesterday_metagraph.validator_trust > 0.0]
+current_vcount = current_validators.sum().item()
+current_mcount = (current_metagraph.trust > 0.0).sum().item()
+yesterday_vcount = yesterday_validators.sum().item()
+yesterday_mcount = (yesterday_metagraph.trust > 0.0).sum().item()
+
+st.markdown('#')    
 
 mcol1, mcol2, mcol3, mcol4 = st.columns(4)
 mcol1.metric('Block', current_block, delta='+7200 [24hr]')
@@ -77,30 +81,64 @@ mcol3.metric('Validators', current_vcount, delta=current_vcount-yesterday_vcount
 mcol4.metric('Miners', current_mcount, delta=current_mcount-yesterday_mcount)
 
 
-st.markdown('#')    
 st.markdown('#')    
 
-bcol1, bcol2, bcol3 = st.columns([0.6, 0.1, 0.2])    
-block_start, block_end = bcol1.select_slider(
-    'Select a **block range**',
-    options=blockfiles,
-    value=(DEFAULT_BLOCK_START, DEFAULT_BLOCK_END),
-    format_func=lambda x: f'{x:,}'
-)
 
-with bcol3:
+with st.sidebar:
+    st.title('Options')
     st.markdown('#')    
-    st.button('Refresh', on_click=run_subprocess)
+    
+    netuid = st.selectbox('Netuid', [1,11], index=0)
 
+    st.markdown('#')    
+
+    c1, c2 = st.columns([0.7,0.3])
+    staleness =  current_block - blockfiles[-1]
+    msg = c1.warning(f'Out of date ({staleness})') if staleness >= 100 else c1.info('Up to date')
+    if c2.button('Update', type='primary'):
+        msg.info('Downloading')      
+        return_code = run_subprocess()
+        if return_code == 0:
+            msg.success('Up to date')
+            time.sleep(1)
+            msg.empty()
+        else:
+            msg.error('Error')
+
+    st.markdown('#')    
+    
+    block_start, block_end = st.select_slider(
+        'Select a **block range**',
+        options=blockfiles,
+        value=(DEFAULT_BLOCK_START, DEFAULT_BLOCK_END),
+        format_func=lambda x: f'{x:,}'
+    )
+    
+    st.markdown('#')    
+    st.markdown('#')    
+    # horizontal line
+    st.markdown('<hr>', unsafe_allow_html=True)
+
+    r1c1, r1c2 = st.columns(2)
+    x = r1c1.selectbox('**Time axis**', ['block','timestamp'], index=0)
+    color = r1c2.selectbox('**Color**', ['coldkey','hotkey','ip'], index=0)
+    r2c1, r2c2 = st.columns(2)
+    ntop = r2c1.slider('**Sample top**', min_value=1, max_value=50, value=10, key='sel_ntop')    
+    opacity = r2c2.slider('**Opacity**', min_value=0., max_value=1., value=0.5, key='opacity')    
+    r3c1, r3c2 = st.columns(2)
+    smooth = r3c1.slider('Smoothing', min_value=1, max_value=100, value=1, key='sel_churn_smooth')
+    smooth_agg = r3c2.radio('Smooth Aggregation', ['mean','std','max','min','sum'], horizontal=True, index=0)
+        
 
 with st.spinner(text=f'Loading data...'):
     # df = load_metagraphs(block_start=block_start, block_end=block_end, block_step=DEFAULT_BLOCK_STEP)
-    df = pd.read_parquet('blocks_600100_807300_100')
+    df = pd.read_parquet(os.path.join(datadir,'df.parquet'))
 
 blocks = df.block.unique()
 
-df_sel = df.loc[df.block.between(block_start, block_end)]
-
+df_sel = df.loc[df.block.between(block_start, block_end)].sort_values(by='block')
+miners = df_sel.loc[df_sel.validator_trust == 0]
+validators = df_sel.loc[df_sel.validator_trust > 0]
 
 # add vertical space
 st.markdown('#')
@@ -108,33 +146,102 @@ st.markdown('#')
 
 tab1, tab2, tab3, tab4 = st.tabs(["Overview", "Miners", "Validators", "Block"])
 
-miner_choices = ['total_stake','ranks','incentive','emission','consensus','trust','validator_trust','dividends']
+validator_choices = ['total_stake','incentive','emission','consensus','validator_trust','dividends']
+miner_choices = ['total_stake','incentive','emission','consensus','trust']
 cabal_choices = ['hotkey','ip','coldkey']
+cabal_choices.remove(color)
 
 ### Overview  ###
 with tab1:
     
-    x_col = st.radio('X-axis', ['block','timestamp'], index=0, horizontal=True)
+    st.markdown('#')
+    st.markdown('#')
+    st.subheader('Hotkey Churn')
+    st.info('**Churn** *measures the change in network participants over time*')
     
-    acol1, acol2 = st.columns([0.3, 0.7])
-    sel_ntop = acol1.slider('Number:', min_value=1, max_value=50, value=10, key='sel_ntop')    
-    #horizontal list
-    miner_choice = acol2.radio('Select:', miner_choices, horizontal=True, index=0)
+    churn_choice = st.radio('Hotkey event', ['changed','added','removed'], horizontal=True, index=0)
+
     st.plotly_chart(
-        plot_trace(df_sel, time_col=x_col,col=miner_choice, ntop=sel_ntop),
+        plotting.plot_churn(df_sel, time_col=x, type=churn_choice, smooth=smooth, smooth_agg=smooth_agg, opacity=opacity),
         use_container_width=True
     )
-
-    col1, col2 = st.columns(2)
-    count_col = col1.radio('Count', cabal_choices, index=0, horizontal=True)
-    y_col = col2.radio('Agg on', cabal_choices, index=2, horizontal=True)
-
+    
+    st.markdown('#')
+    st.markdown('#')
+    st.subheader('Network Occupancy')
+    st.info('**Occupancy** *measures the number of network participants at a given time*')    
     st.plotly_chart(
-        plot_cabals(df_sel, time_col=x_col, count_col=count_col, sel_col=y_col, ntop=sel_ntop), 
+        plotting.plot_occupancy(df_sel, time_col=x, smooth=smooth, smooth_agg=smooth_agg, opacity=opacity),
         use_container_width=True
     )
 
+animation_aggs = ['mean','sum','std','max','min']
+mac_choices = [f'{col}_{agg}' for col in miner_choices for agg in animation_aggs]+['hotkey_nunique','ip_nunique']
+vac_choices = [f'{col}_{agg}' for col in validator_choices for agg in animation_aggs]+['hotkey_nunique','ip_nunique']
+
 with tab2:
     
-    # plot of miner weights versus time/block
-    pass
+    st.markdown('#')
+    st.markdown('#')
+    st.subheader('Miner Activity')
+    st.info('**Activity** *shows the change in stake and emission over time for **miners**, grouped by coldkey*')
+    
+    mac1, mac2, mac3, mac4 = st.columns(4)
+    mac_x = mac1.selectbox('**x**', mac_choices, index=mac_choices.index('emission_mean'))
+    mac_y = mac2.selectbox('**y**', mac_choices, index=mac_choices.index('trust_mean'))
+    mac_size = mac3.selectbox('**marker size**', mac_choices, index=mac_choices.index('hotkey_nunique'))
+    mac_color = mac4.selectbox('**marker color**', mac_choices, index=mac_choices.index('total_stake_sum'))
+
+    st.plotly_chart(
+        plotting.plot_animation(miners, x=mac_x, y=mac_y, color=mac_color, size=mac_size, opacity=opacity),
+        use_container_width=True
+    )    
+    
+    miner_choice = st.radio('Select:', miner_choices, horizontal=True, index=0)
+    with st.expander(f'Show **{miner_choice}** trends for top **{ntop}** miners'):
+    
+        st.plotly_chart(
+            plotting.plot_trace(miners, time_col=x, col=miner_choice, ntop=ntop, smooth=smooth, smooth_agg=smooth_agg, opacity=opacity),
+            use_container_width=True
+        )
+
+    count_col = st.radio('Count', cabal_choices, index=0, horizontal=True, key='sel_miner_count')
+    with st.expander(f'Show **{count_col}** trends for top **{ntop}** miners'):
+
+        st.plotly_chart(
+            plotting.plot_cabals(miners, time_col=x, count_col=count_col, sel_col=color, ntop=ntop, smooth=smooth, smooth_agg=smooth_agg, opacity=opacity), 
+            use_container_width=True
+        )
+
+with tab3:
+    
+    st.markdown('#')
+    st.markdown('#')
+    st.subheader('Validator Activity')
+    st.info('**Activity** *shows the change in stake and emission over time for **validators**, grouped by coldkey*')
+    
+    vac1, vac2, vac3, vac4 = st.columns(4)
+    vac_x = vac1.selectbox('**x**', vac_choices, index=vac_choices.index('incentive_mean'))
+    vac_y = vac2.selectbox('**y**', vac_choices, index=vac_choices.index('validator_trust_mean'))
+    vac_size = vac3.selectbox('**marker size**', vac_choices, index=vac_choices.index('hotkey_nunique'))
+    vac_color = vac4.selectbox('**marker color**', vac_choices, index=vac_choices.index('total_stake_sum'))
+    
+    st.plotly_chart(
+        plotting.plot_animation(validators, x=vac_x, y=vac_y, color=vac_color, size=vac_size, opacity=opacity),
+        use_container_width=True
+    )    
+    validator_choice = st.radio('Select:', validator_choices, horizontal=True, index=0)
+    with st.expander(f'Show **{validator_choice}** trends for top **{ntop}** validators'):
+    
+        st.plotly_chart(
+            plotting.plot_trace(validators, time_col=x,col=validator_choice, ntop=ntop, smooth=smooth, smooth_agg=smooth_agg, opacity=opacity),
+            use_container_width=True
+        )
+
+    count_col = st.radio('Count', cabal_choices, index=0, horizontal=True, key='sel_validator_count')
+    with st.expander(f'Show **{count_col}** trends for top **{ntop}** validators'):
+
+        st.plotly_chart(
+            plotting.plot_cabals(validators, time_col=x, count_col=count_col, sel_col=color, ntop=ntop, smooth=smooth, smooth_agg=smooth_agg, opacity=opacity), 
+            use_container_width=True
+        )

multigraph.py

@@ -115,5 +115,19 @@ if __name__ == '__main__':
 
 
     if not args.no_dataframe:
-        df = load_metagraphs(min(blocks), max(blocks), block_step=step_size, datadir=datadir)
-        df.to_pickle(f'data/metagraph/{netuid}/df.pkl')
+        save_path = f'data/metagraph/{netuid}/df.parquet'
+        blocks = range(start_block, end_block, step_size)
+        df_loaded = None
+        if os.path.exists(save_path):
+            df_loaded = pd.read_parquet(save_path)
+            blocks = [block for block in blocks if block not in df_loaded.block.unique()]
+            print(f'Loaded dataframe from {save_path!r}. {len(df_loaded)} rows. {len(blocks)} blocks to process.')
+            if len(blocks)==0:
+                print('No blocks to process.')
+                sys.exit(0)
+            
+        df = load_metagraphs(blocks[0], blocks[-1], block_step=step_size, datadir=datadir)
+        if df_loaded is not None:
+            df = pd.concat([df, df_loaded], ignore_index=True)
+        df.to_parquet(save_path)
+        print(f'Saved dataframe to {save_path!r}')