Update metagraph dashboard

meta_utils.py

@@ -39,7 +39,7 @@ def load_metagraphs(block_start, block_end, block_step=1000, datadir='data/metag
 
     blocks = range(block_start, block_end, block_step)
     print(f'Loading blocks {blocks[0]}-{blocks[-1]} from {datadir}')
-    filenames = sorted(filename for filename in os.listdir(datadir) if int(filename.split('.')[0]) in blocks)
+    filenames = sorted(filename for filename in os.listdir(datadir) if filename.split('.')[0].isdigit() and int(filename.split('.')[0]) in blocks)
     print(f'Found {len(filenames)} files in {datadir}')
     
     metagraphs = []

metadash.py

@@ -5,6 +5,7 @@ from meta_utils import run_subprocess, load_metagraphs
 # from opendashboards.assets import io, inspect, metric, plot
 from meta_plotting import plot_trace, plot_cabals
 import asyncio
+from functools import lru_cache
 
 ## TODO: Read blocks from a big parquet file instead of loading all the pickles -- this is slow
 
@@ -47,24 +48,49 @@ st.title('Metagraph :red[Analysis] Dashboard :eyes:')
 st.markdown('#')
 st.markdown('#')
 
+netuid = 1
 subtensor = bittensor.subtensor(network='finney')
 current_block = subtensor.get_current_block()
-current_difficulty = subtensor.difficulty(1, block=current_block)
 
-bcol1, bcol2, bcol3 = st.columns([0.2, 0.6, 0.2])
-with bcol1:
-    st.metric('Current **block**', current_block, delta='+7200 [24hr]')
-    # st.metric('Current **difficulty**', f'{current_difficulty/10e12:.0}T', delta='?')
+@lru_cache(maxsize=1)
+def _metagraph(block):
     
+    print('rerunning cache')
+    return ( 
+            subtensor.metagraph(netuid, block=block), 
+            subtensor.metagraph(netuid, block=block - 7200),
+            subtensor.burn(netuid=netuid, block=block),
+            subtensor.burn(netuid=netuid, block=block - 7200),
+    )
     
-block_start, block_end = bcol2.select_slider(
+current_metagraph, yesterday_metagraph, current_burn, yesterday_burn = _metagraph(10*current_block//10)
+
+current_vcount = current_metagraph.validator_permit.sum().item()
+current_mcount = (~current_metagraph.validator_permit).sum().item()
+yesterday_vcount = yesterday_metagraph.validator_permit.sum().item()
+yesterday_mcount = (~yesterday_metagraph.validator_permit).sum().item()
+
+mcol1, mcol2, mcol3, mcol4 = st.columns(4)
+mcol1.metric('Block', current_block, delta='+7200 [24hr]')
+mcol2.metric('Register Cost', f'{current_burn.unit}{current_burn.tao:.3f}', delta=f'{current_burn.tao-yesterday_burn.tao:.3f}')
+mcol3.metric('Validators', current_vcount, delta=current_vcount-yesterday_vcount)
+mcol4.metric('Miners', current_mcount, delta=current_mcount-yesterday_mcount)
+
+
+st.markdown('#')    
+st.markdown('#')    
+
+bcol1, bcol2, bcol3 = st.columns([0.6, 0.1, 0.2])    
+block_start, block_end = bcol1.select_slider(
     'Select a **block range**',
     options=blockfiles,
     value=(DEFAULT_BLOCK_START, DEFAULT_BLOCK_END),
     format_func=lambda x: f'{x:,}'
 )
 
-bcol3.button('Refresh', on_click=run_subprocess)
+with bcol3:
+    st.markdown('#')    
+    st.button('Refresh', on_click=run_subprocess)
 
 
 with st.spinner(text=f'Loading data...'):

multigraph.py

@@ -9,7 +9,7 @@ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
 
 import torch
 import bittensor
-
+from meta_utils import load_metagraphs
 #TODO: make line charts and other cool stuff for each metagraph snapshot
 
 def process(block, netuid=1, lite=True, difficulty=False, prune_weights=False, return_graph=False, half=True, subtensor=None):
@@ -43,8 +43,9 @@ def parse_arguments():
     parser.add_argument('--difficulty', action='store_true', help='Include difficulty in metagraph.')
     parser.add_argument('--prune_weights', action='store_true', help='Prune weights in metagraph.')
     parser.add_argument('--return_graph', action='store_true', help='Return metagraph instead of True.')
+    parser.add_argument('--no_dataframe', action='store_true', help='Do not create dataframe.')
     parser.add_argument('--max_workers', type=int, default=32, help='Max workers to use.')
-    parser.add_argument('--start_block', type=int, default=1_000_000, help='Start block.')
+    parser.add_argument('--start_block', type=int, default=1_500_000, help='Start block.')
     parser.add_argument('--end_block', type=int, default=600_000, help='End block.')
     parser.add_argument('--step_size', type=int, default=100, help='Step size.')
     return parser.parse_args()
@@ -72,41 +73,47 @@ if __name__ == '__main__':
     
     max_workers = min(args.max_workers, len(blocks))
 
-    os.makedirs(f'data/metagraph/{netuid}', exist_ok=True)
+    datadir = f'data/metagraph/{netuid}'
+    os.makedirs(datadir, exist_ok=True)
     if not overwrite:
         blocks = [block for block in blocks if not os.path.exists(f'data/metagraph/{netuid}/{block}.pkl')]
 
     metagraphs = []
     
-    if len(blocks)==0:
-        print(f'No blocks to process. Current block: {subtensor.block}')
-        quit()
+    if len(blocks)>0:
         
-    print(f'Processing {len(blocks)} blocks from {blocks[0]}-{blocks[-1]} using {max_workers} workers.')
-    
-    with ProcessPoolExecutor(max_workers=max_workers) as executor:
-        futures = [
-            executor.submit(process, block, lite=lite(block), netuid=netuid, difficulty=difficulty) 
-            for block in blocks
-            ]
-
-        success = 0        
-        with tqdm.tqdm(total=len(futures)) as pbar:
-            for block, future in zip(blocks,futures):
-                try:
-                    metagraphs.append(future.result())
-                    success += 1
-                except Exception as e:
-                    print(f'generated an exception: {print_exc(e)}')
-                pbar.update(1)
-                pbar.set_description(f'Processed {success} blocks. Current block: {block}')
-
-    if not success:
-        raise ValueError('No blocks were successfully processed.')
-    
-    print(f'Processed {success} blocks.')
-    if return_graph:
-        for metagraph in metagraphs:
-            print(f'{metagraph.block}: {metagraph.n.item()} nodes, difficulty={getattr(metagraph, "difficulty", None)}, weights={metagraph.weights.shape if hasattr(metagraph, "weights") else None}')
+        print(f'Processing {len(blocks)} blocks from {blocks[0]}-{blocks[-1]} using {max_workers} workers.')
+        
+        with ProcessPoolExecutor(max_workers=max_workers) as executor:
+            futures = [
+                executor.submit(process, block, lite=lite(block), netuid=netuid, difficulty=difficulty) 
+                for block in blocks
+                ]
+
+            success = 0        
+            with tqdm.tqdm(total=len(futures)) as pbar:
+                for block, future in zip(blocks,futures):
+                    try:
+                        metagraphs.append(future.result())
+                        success += 1
+                    except Exception as e:
+                        print(f'generated an exception: {print_exc(e)}')
+                    pbar.update(1)
+                    pbar.set_description(f'Processed {success} blocks. Current block: {block}')
+
+        if not success:
+            raise ValueError('No blocks were successfully processed.')
+        
+        print(f'Processed {success} blocks.')
+        if return_graph:
+            for metagraph in metagraphs:
+                print(f'{metagraph.block}: {metagraph.n.item()} nodes, difficulty={getattr(metagraph, "difficulty", None)}, weights={metagraph.weights.shape if hasattr(metagraph, "weights") else None}')
+
+        print(metagraphs[-1])
+    else:
+        print(f'No blocks to process. Current block: {subtensor.block}')
+
 
-    print(metagraphs[-1])
+    if not args.no_dataframe:
+        df = load_metagraphs(min(blocks), max(blocks), block_step=step_size, datadir=datadir)
+        df.to_pickle(f'data/metagraph/{netuid}/df.pkl')