using historical data from otf wandb and current data

utils.py

@@ -15,7 +15,7 @@ import bittensor as bt
 MIN_STEPS = 12 # minimum number of steps in wandb run in order to be worth analyzing
 MAX_RUNS = 100#0000
 NETUID = 25
-BASE_PATH = 'macrocosmos/folding-validators--moved'
+BASE_PATHS = ['macrocosmos/folding-validators--moved', 'macrocosmos/folding-validators'] # added historical data from otf wandb and current data
 NETWORK = 'finney'
 KEYS = None
 ABBREV_CHARS = 8
@@ -25,7 +25,7 @@ PDBS_PER_RUN_STEP = 0.083
 AVG_MD_STEPS = 30_000
 BASE_UNITS = 'MB'
 
-api = wandb.Api(timeout=120, api_key=os.environ['WANDB_API_KEY'])
+api = wandb.Api(timeout=120, api_key='cdcbe340bb7937d3a289d39632491d12b39231b7')
 
 IDENTITIES = {
     '5F4tQyWrhfGVcNhoqeiNsR6KjD4wMZ2kfhLj4oHYuyHbZAc3': 'opentensor',
@@ -201,7 +201,7 @@ def load_run(run_path, keys=KEYS):
     return df
 
 @st.cache_data(show_spinner=False)
-def build_data(timestamp=None, path=BASE_PATH, min_steps=MIN_STEPS, use_cache=True):
+def build_data(timestamp=None, paths=BASE_PATHS, min_steps=MIN_STEPS, use_cache=True):
 
     save_path = '_saved_runs.csv'
     filters = {}
@@ -216,20 +216,24 @@ def build_data(timestamp=None, path=BASE_PATH, min_steps=MIN_STEPS, use_cache=Tr
         filters.update({'updated_at': {'$gte': timestamp_str}})
 
     progress = st.progress(0, text='Loading data')
+    
+    historical_runs = api.runs(paths[0], filters=filters)
+    historical_and_current_runs = [historical_runs, api.runs(paths[1], filters=filters)]
 
-    runs = api.runs(path, filters=filters)
-
+    
     run_data = []
     n_events = 0
-    for i, run in enumerate(tqdm.tqdm(runs, total=len(runs))):
-        num_steps = run.summary.get('_step',0)
-        if num_steps<min_steps:
-            continue
-        n_events += num_steps
-        prog_msg = f'Loading data {i/len(runs)*100:.0f}%, {n_events:,.0f} events)'
-        progress.progress(i/len(runs),text=f'{prog_msg}... **downloading** `{os.path.join(*run.path)}`')
-
-        run_data.append(run)
+    total_runs = len(historical_and_current_runs[0])+len(historical_and_current_runs[1])
+    for runs in historical_and_current_runs:
+        for i, run in enumerate(tqdm.tqdm(runs, total=total_runs)):
+            num_steps = run.summary.get('_step',0)
+            if num_steps<min_steps:
+                continue
+            n_events += num_steps
+            prog_msg = f'Loading data {i/total_runs*100:.0f}%, {n_events:,.0f} events)'
+            progress.progress(i/total_runs,text=f'{prog_msg}... **downloading** `{os.path.join(*run.path)}`')
+
+            run_data.append(run)
 
     progress.empty()