Refactor package structure, some redesign and improved layout

opendashboards/__init__.py

@@ -0,0 +1,3 @@
+
+
+__version__ = "0.1.0"

opendashboards/assets/inspect.py

@@ -0,0 +1,55 @@
+
+import streamlit as st
+import pandas as pd
+import opendashboards.utils.utils as utils
+
+
+@st.cache_data
+def explode_data(df):
+    list_cols = utils.get_list_col_lengths(df)
+    try:
+        return utils.explode_data(df, list(list_cols.keys())).apply(pd.to_numeric, errors='ignore')
+    except Exception as e:
+        st.error(f'Error exploding data with columns')
+        st.write(list_cols)
+        st.exception(e)
+        st.dataframe(df)
+        st.stop()
+
+@st.cache_data
+def completions(df_long, col):
+    return df_long[col].value_counts()
+
+@st.cache_data
+def weights(df, index='_timestamp'):
+    # Create a column for each UID and show most recent rows
+    scores = df['moving_averaged_scores'].apply(pd.Series).fillna(method='ffill')
+    if index in df.columns:
+        scores.index = df[index]
+
+    # rename columns
+    scores.rename({i: f'UID-{i}' for i in range(scores.shape[1])}, axis=1, inplace=True)
+    return scores    
+    
+def run_event_data(df_runs, df, selected_runs):
+    
+    st.markdown('#')
+
+    show_col1, show_col2 = st.columns(2)
+    show_runs = show_col1.checkbox('Show runs', value=True)
+    show_events = show_col2.checkbox('Show events', value=False)
+    if show_runs:
+        st.markdown(f'Wandb info for **{len(selected_runs)} selected runs**:')
+        st.dataframe(df_runs.loc[df_runs.id.isin(selected_runs)],
+                    column_config={
+                        "url": st.column_config.LinkColumn("URL"),
+                    }
+        )
+
+    if show_events:
+        st.markdown(f'Raw events for **{len(selected_runs)} selected runs**:')
+        st.dataframe(df.head(50),
+                    column_config={
+                        "url": st.column_config.LinkColumn("URL"),
+                    }
+        )

opendashboards/assets/io.py

@@ -0,0 +1,95 @@
+import os
+import re
+import pandas as pd
+import streamlit as st
+
+import  opendashboards.utils.utils as utils
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+@st.cache_data
+def load_runs(project, filters, min_steps=10):
+    runs = []
+    msg = st.empty()
+    for run in utils.get_runs(project, filters):
+        step = run.summary.get('_step',0)
+        if step < min_steps:
+            msg.warning(f'Skipped run `{run.name}` because it contains {step} events (<{min_steps})')
+            continue
+
+        duration = run.summary.get('_runtime')
+        end_time = run.summary.get('_timestamp')
+        # extract values for selected tags
+        rules = {'hotkey': re.compile('^[0-9a-z]{48}$',re.IGNORECASE), 'version': re.compile('^\\d\.\\d+\.\\d+$'), 'spec_version': re.compile('\\d{4}$')}
+        tags = {k: tag for k, rule in rules.items() for tag in run.tags if rule.match(tag)}
+        # include bool flag for remaining tags
+        tags.update({k: k in run.tags for k in ('mock','custom_gating_model','nsfw_filter','outsource_scoring','disable_set_weights')})
+
+        runs.append({
+            'state': run.state,
+            'num_steps': step,
+            'entity': run.entity,
+            'id': run.id,
+            'name': run.name,
+            'project': run.project,
+            'url': run.url,
+            'path': os.path.join(run.entity, run.project, run.id),
+            'start_time': pd.to_datetime(end_time-duration, unit="s"),
+            'end_time': pd.to_datetime(end_time, unit="s"),
+            'duration': pd.to_datetime(duration, unit="s"),
+            **tags
+        })
+    msg.empty()
+    return pd.DataFrame(runs).astype({'state': 'category', 'hotkey': 'category', 'version': 'category', 'spec_version': 'category'})
+
+
+@st.cache_data
+def load_data(selected_runs, load=True, save=False):
+
+    frames = []
+    n_events = 0
+    progress = st.progress(0, 'Loading data')
+    info = st.empty()
+    for i, idx in enumerate(selected_runs.index):
+        run = selected_runs.loc[idx]
+        prog_msg = f'Loading data {i/len(selected_runs)*100:.0f}% ({i}/{len(selected_runs)} runs, {n_events} events)'
+
+        rel_path = os.path.join('data',f'history-{run.id}.csv')
+        file_path = os.path.join(BASE_DIR,rel_path)
+
+        if load and os.path.exists(file_path):
+            progress.progress(i/len(selected_runs),f'{prog_msg}... **reading** `{rel_path}`')
+            try:
+                df = utils.load_data(file_path)
+            except Exception as e:
+                info.warning(f'Failed to load history from `{file_path}`')
+                st.exception(e)
+                continue
+        else:
+            progress.progress(i/len(selected_runs),f'{prog_msg}... **downloading** `{run.path}`')
+            try:
+                # Download the history from wandb
+                df = utils.download_data(run.path)
+                df.assign(**run.to_dict())
+                if not os.path.exists('data/'):
+                    os.makedirs(file_path)
+
+                if save and run.state != 'running':
+                    df.to_csv(file_path, index=False)
+                    # st.info(f'Saved history to {file_path}')
+            except Exception as e:
+                info.warning(f'Failed to download history for `{run.path}`')
+                st.exception(e)
+                continue
+
+        frames.append(df)
+        n_events += df.shape[0]
+
+    progress.empty()
+    if not frames:
+        info.error('No data loaded')
+        st.stop()
+    # Remove rows which contain chain weights as it messes up schema
+    return pd.concat(frames)
+
+

opendashboards/assets/metric.py

@@ -0,0 +1,60 @@
+import os
+import re
+import time
+import pandas as pd
+import streamlit as st
+
+
+@st.cache_data
+def wandb(df_runs):
+
+    # get rows where start time is older than 24h ago
+    df_runs_old = df_runs.loc[df_runs.start_time < pd.to_datetime(time.time()-24*60*60, unit='s')]
+
+    col1, col2, col3 = st.columns(3)
+
+    col1.metric('Runs', df_runs.shape[0], delta=f'{df_runs.shape[0]-df_runs_old.shape[0]} (24h)')
+    col2.metric('Hotkeys', df_runs.hotkey.nunique(), delta=f'{df_runs.hotkey.nunique()-df_runs_old.hotkey.nunique()} (24h)')
+    col3.metric('Events', df_runs.num_steps.sum(), delta=f'{df_runs.num_steps.sum()-df_runs_old.num_steps.sum()} (24h)')
+    st.markdown('----')
+
+
+@st.cache_data
+def runs(df, df_long, selected_runs):
+
+    col1, col2, col3 = st.columns(3)
+    col1.metric(label="Runs", value=len(selected_runs))
+    col1.metric(label="Events", value=df.shape[0]) #
+    col2.metric(label="Followup UIDs", value=df_long.followup_uids.nunique())
+    col2.metric(label="Answer UIDs", value=df_long.answer_uids.nunique())
+    col3.metric(label="Followup Completions", value=df_long.followup_completions.nunique())
+    col3.metric(label="Answer Completions", value=df_long.answer_completions.nunique())
+    st.markdown('----')
+
+
+    
+@st.cache_data
+def uids(df_long, src, uid=None):
+
+    uid_col = f'{src}_uids'
+    completion_col = f'{src}_completions'
+    nsfw_col = f'{src}_nsfw_scores'
+    reward_col = f'{src}_rewards'
+
+    if uid is not None:
+        df_long = df_long.loc[df_long[uid_col] == uid]
+
+    col1, col2, col3 = st.columns(3)
+    col1.metric(
+        label="Success %",
+        value=f'{df_long.loc[df_long[completion_col].str.len() > 0].shape[0]/df_long.shape[0] * 100:.1f}'
+    )
+    col2.metric(
+        label="Diversity %",
+        value=f'{df_long[completion_col].nunique()/df_long.shape[0] * 100:.1f}'
+    )
+    col3.metric(
+        label="Toxicity %",
+        value=f'{df_long[nsfw_col].mean() * 100:.1f}' if nsfw_col in df_long.columns else 'N/A'
+    )
+    st.markdown('----')

opendashboards/assets/plot.py

@@ -0,0 +1,52 @@
+
+import streamlit as st
+import utils.plotting as plotting
+
+# @st.cache_data
+def uid_diversty(df, rm_failed=True):
+    return st.plotly_chart(
+        plotting.plot_uid_diversty(
+            df,
+            remove_unsuccessful=rm_failed
+        ),
+        use_container_width=True
+    )
+
+# @st.cache_data
+def leaderboard(df, ntop, group_on, agg_col, agg, alias=False):
+    return st.plotly_chart(
+        plotting.plot_leaderboard(
+            df,
+            ntop=ntop,
+            group_on=group_on,
+            agg_col=agg_col,
+            agg=agg,
+            alias=alias
+        ),
+        use_container_width=True
+    )
+
+# @st.cache_data
+def completion_rewards(df, completion_col, reward_col, uid_col, ntop, completions=None, completion_regex=None):
+    return st.plotly_chart(
+        plotting.plot_completion_rewards(
+            df,
+            msg_col=completion_col,
+            reward_col=reward_col,
+            uid_col=uid_col,
+            ntop=ntop,
+            completions=completions,
+            completion_regex=completion_regex
+        ),
+        use_container_width=True
+    )
+    
+def weights(df, uids, ntop=10):
+    return st.plotly_chart(
+        plotting.plot_weights(
+            df,
+            uids=[f'UID-{i}' for i in uids],
+            ntop=ntop
+        ),
+        use_container_width=True
+    )

opendashboards/dashboard.py

@@ -1,148 +1,29 @@
 import streamlit as st
-import pandas as pd
-import os
-import re
-import time
-from .utils import get_runs, download_data, get_list_col_lengths, explode_data
-import .plotting as plotting
-
+from opendashboards.assets import io, inspect, metric, plot
 
 # dendrite time versus completion length
 # prompt-based completion score stats
-
+# instrospect specific RUN-UID-COMPLETION
 
 DEFAULT_PROJECT = "openvalidators"
 DEFAULT_FILTERS = {"tags": {"$in": ["1.0.0", "1.0.1", "1.0.2", "1.0.3", "1.0.4"]}}
-
-@st.cache_data
-def load_runs(project=DEFAULT_PROJECT, filters=DEFAULT_FILTERS, min_steps=10):
-    runs = []
-    msg = st.empty()
-    for run in get_runs(project, filters):
-        step = run.summary.get('_step',0)
-        if step < min_steps:
-            msg.warning(f'Skipped run `{run.name}` because it contains {step} events (<{min_steps})')
-            continue
-
-        duration = run.summary.get('_runtime')
-        end_time = run.summary.get('_timestamp')
-        # extract values for selected tags
-        rules = {'hotkey': re.compile('^[0-9a-z]{48}$',re.IGNORECASE), 'version': re.compile('^\\d\.\\d+\.\\d+$'), 'spec_version': re.compile('\\d{4}$')}
-        # tags = {k: tag for k,tag in zip(('hotkey','version','spec_version'),run.tags)}
-        tags = {k: tag for k, rule in rules.items() for tag in run.tags if rule.match(tag)}
-        # include bool flag for remaining tags
-        tags.update({k: k in run.tags for k in ('mock','custom_gating_model','nsfw_filter','outsource_scoring','disable_set_weights')})
-
-        runs.append({
-            'state': run.state,
-            'num_steps': step,
-            'entity': run.entity,
-            'id': run.id,
-            'name': run.name,
-            'project': run.project,
-            'url': run.url,
-            'path': os.path.join(run.entity, run.project, run.id),
-            'start_time': pd.to_datetime(end_time-duration, unit="s"),
-            'end_time': pd.to_datetime(end_time, unit="s"),
-            'duration': pd.to_datetime(duration, unit="s"),
-            # 'tags': run.tags,
-            **tags
-        })
-    msg.empty()
-    return pd.DataFrame(runs).astype({'state': 'category', 'hotkey': 'category', 'version': 'category', 'spec_version': 'category'})
-
-
-@st.cache_data
-def load_data(selected_runs, load=True, save=False):
-
-    frames = []
-    n_events = 0
-    progress = st.progress(0, 'Loading data')
-    for i, idx in enumerate(selected_runs.index):
-        run = selected_runs.loc[idx]
-        prog_msg = f'Loading data {i/len(selected_runs)*100:.0f}% ({i}/{len(selected_runs)} runs, {n_events} events)'
-
-        file_path = f'data/history-{run.id}.csv'
-
-        if load and os.path.exists(file_path):
-            progress.progress(i/len(selected_runs),f'{prog_msg}... reading {file_path}')
-            df = pd.read_csv(file_path)
-            # filter out events with missing step length
-            df = df.loc[df.step_length.notna()]
-
-            # detect list columns which as stored as strings
-            list_cols = [c for c in df.columns if df[c].dtype == "object" and df[c].str.startswith("[").all()]
-            # convert string representation of list to list
-            df[list_cols] = df[list_cols].applymap(eval, na_action='ignore')
-
-        else:
-            try:
-                # Download the history from wandb
-                progress.progress(i/len(selected_runs),f'{prog_msg}... downloading `{run.path}`')
-                df = download_data(run.path)
-                df.assign(**run.to_dict())
-                if not os.path.exists('data/'):
-                    os.makedirs(file_path)
-
-                if save and run.state != 'running':
-                    df.to_csv(file_path, index=False)
-                    # st.info(f'Saved history to {file_path}')
-            except Exception as e:
-                st.error(f'Failed to download history for `{run.path}`')
-                st.exception(e)
-                continue
-
-        frames.append(df)
-        n_events += df.shape[0]
-
-    progress.empty()
-    # Remove rows which contain chain weights as it messes up schema
-    return pd.concat(frames)
-
-@st.cache_data
-def get_exploded_data(df):
-    list_cols = get_list_col_lengths(df)
-    return explode_data(df, list(list_cols))
-
-@st.cache_data
-def get_completions(df_long, col):
-    return df_long[col].value_counts()
-
-@st.cache_data
-def plot_uid_diversty(df, remove_unsuccessful=True):
-    return plotting.plot_uid_diversty(df, remove_unsuccessful=remove_unsuccessful)
-
-@st.cache_data
-def plot_leaderboard(df, ntop, group_on, agg_col, agg, alias=False):
-    return plotting.plot_leaderboard(df, ntop=ntop, group_on=group_on, agg_col=agg_col, agg=agg, alias=alias)
-
-@st.cache_data
-def plot_completion_rewards(df, completion_col, reward_col, uid_col, ntop, completions=None, completion_regex=None):
-    return plotting.plot_completion_rewards(df, msg_col=completion_col, reward_col=reward_col, uid_col=uid_col, ntop=ntop, completions=completions, completion_regex=completion_regex)
-
-@st.cache_data
-def uid_metrics(df_long, src, uid=None):
-
-    uid_col = f'{src}_uids'
-    completion_col = f'{src}_completions'
-    nsfw_col = f'{src}_nsfw_scores'
-    reward_col = f'{src}_rewards'
-
-    if uid is not None:
-        df_long = df_long.loc[df_long[uid_col] == uid]
-
-    col1, col2, col3 = st.columns(3)
-    col1.metric(
-        label="Success %",
-        value=f'{df_long.loc[df_long[completion_col].str.len() > 0].shape[0]/df_long.shape[0] * 100:.1f}'
-    )
-    col2.metric(
-        label="Diversity %",
-        value=f'{df_long[completion_col].nunique()/df_long.shape[0] * 100:.1f}'
-    )
-    col3.metric(
-        label="Toxicity %",
-        value=f'{df_long[nsfw_col].mean() * 100:.1f}' if nsfw_col in df_long.columns else 'N/A'
+DEFAULT_SELECTED_RUNS = ['kt9bzxii']
+DEFAULT_SRC = 'followup'
+DEFAULT_COMPLETION_NTOP = 10
+DEFAULT_UID_NTOP = 10
+
+# Set app config
+st.set_page_config(
+    page_title='Validator Dashboard',
+    menu_items={
+        'Report a bug': "https://github.com/opentensor/dashboards/issues",
+        'About': """
+        This dashboard is part of the OpenTensor project. \n
+        To see runs in wandb, go to: \n
+        https://wandb.ai/opentensor-dev/openvalidators/table?workspace=default
+        """
+    },
+    layout = "centered"
     )
 
 st.title('Validator :red[Analysis] Dashboard :eyes:')
@@ -150,155 +31,132 @@ st.title('Validator :red[Analysis] Dashboard :eyes:')
 st.markdown('#')
 st.markdown('#')
 
-
-with st.sidebar:
-    st.sidebar.header('Pages')
+# with st.sidebar:
+#     st.sidebar.header('Pages')
 
 with st.spinner(text=f'Checking wandb...'):
-    df_runs = load_runs()
-# get rows where start time is older than 24h ago
-df_runs_old = df_runs.loc[df_runs.start_time < pd.to_datetime(time.time()-24*60*60, unit='s')]
-
-col1, col2, col3 = st.columns(3)
-
-col1.metric('Runs', df_runs.shape[0], delta=f'{df_runs.shape[0]-df_runs_old.shape[0]} (24h)')
-col2.metric('Hotkeys', df_runs.hotkey.nunique(), delta=f'{df_runs.hotkey.nunique()-df_runs_old.hotkey.nunique()} (24h)')
-col3.metric('Events', df_runs.num_steps.sum(), delta=f'{df_runs.num_steps.sum()-df_runs_old.num_steps.sum()} (24h)')
-
-# https://wandb.ai/opentensor-dev/openvalidators/runs/kt9bzxii/overview?workspace=
-# all_run_paths = ['opentensor-dev/openvalidators/kt9bzxii'] # pedro long run
+    df_runs = io.load_runs(project=DEFAULT_PROJECT, filters=DEFAULT_FILTERS, min_steps=10)
 
-run_ids = df_runs.id
-default_selected_runs = ['kt9bzxii']
-selected_runs = default_selected_runs
+metric.wandb(df_runs)
 
 # add vertical space
 st.markdown('#')
 st.markdown('#')
 
-
 tab1, tab2, tab3, tab4 = st.tabs(["Wandb Runs", "UID Health", "Completions", "Prompt-based scoring"])
 
-# src = st.radio('Choose data source:', ['followup', 'answer'], horizontal=True, key='src')
-# list_list_cols = get_list_col_lengths(df_long)
-# df_long_long = explode_data(df_long, list(list_list_cols))
 
+### Wandb Runs ###
 with tab1:
 
     st.markdown('#')
-    st.subheader(":violet[Wandb] Runs")
+    st.header(":violet[Wandb] Runs")
 
-    # Load data
-    df = load_data(df_runs.loc[run_ids.isin(selected_runs)], load=True, save=True)
-    df_long = get_exploded_data(df)
+    run_msg = st.info("Select a single run or compare multiple runs")
+    selected_runs = st.multiselect(f'Runs ({len(df_runs)})', df_runs.id, default=DEFAULT_SELECTED_RUNS, key='runs')
 
-    col1, col2, col3, col4 = st.columns(4)
-    col1.metric(label="Selected runs", value=len(selected_runs))
-    col2.metric(label="Events", value=df.shape[0]) #
-    col3.metric(label="UIDs", value=df_long.followup_uids.nunique())
-    col4.metric(label="Unique completions", value=df_long.followup_uids.nunique())
+    # Load data if new runs selected
+    if not selected_runs:
+        # open a dialog to select runs
+        run_msg.error("Please select at least one run")
+        st.snow()
+        st.stop()
 
-    selected_runs = st.multiselect(f'Runs ({len(df_runs)})', run_ids, default=selected_runs)
+    df = io.load_data(df_runs.loc[df_runs.id.isin(selected_runs)], load=True, save=True)
+    df_long = inspect.explode_data(df)
+    df_weights = inspect.weights(df)
 
-    st.markdown('#')
-    st.subheader("View :violet[Data]")
-
-    show_col1, show_col2 = st.columns(2)
-    show_runs = show_col1.checkbox('Show runs', value=True)
-    show_events = show_col2.checkbox('Show events', value=False)
-    if show_runs:
-        st.markdown(f'Wandb info for **{len(selected_runs)} selected runs**:')
-        st.dataframe(df_runs.loc[run_ids.isin(selected_runs)],
-                    column_config={
-                        "url": st.column_config.LinkColumn("URL"),
-                    }
-        )
+    metric.runs(df, df_long, selected_runs)
+
+    with st.expander(f'Show :violet[raw] data for {len(selected_runs)} selected runs'):
+        inspect.run_event_data(df_runs,df, selected_runs)
 
-    if show_events:
-        st.markdown(f'Raw events for **{len(selected_runs)} selected runs**:')
-        st.dataframe(df.head(50),
-                    column_config={
-                        "url": st.column_config.LinkColumn("URL"),
-                    }
-        )
 
-default_src = 'followup'
+### UID Health ###
 with tab2:
 
     st.markdown('#')
-    st.subheader("UID :violet[Health]")
-    uid_src = default_src
+    st.header("UID :violet[Health]")
+    st.info(f"Showing UID health metrics for **{len(selected_runs)} selected runs**")
 
-    # uid = st.selectbox('UID:', sorted(df_long[uid_col].unique()), key='uid')
-
-    uid_metrics(df_long, uid_src)
     uid_src = st.radio('Select one:', ['followup', 'answer'], horizontal=True, key='uid_src')
-    uid_col = f'{uid_src}_uids'
-    reward_col = f'{uid_src}_rewards'
 
-    st.markdown('#')
-    st.subheader("UID :violet[Leaderboard]")
-    uid_ntop_default = 10
+    metric.uids(df_long, uid_src)
 
-    uid_col1, uid_col2 = st.columns(2)
-    uid_ntop = uid_col1.slider('Number of UIDs:', min_value=1, max_value=50, value=uid_ntop_default, key='uid_ntop')
-    uid_agg = uid_col2.selectbox('Aggregation:', ('mean','min','max','size','nunique'), key='uid_agg')
+    with st.expander(f'Show UID **{uid_src}** weights data for **{len(selected_runs)} selected runs**'):
 
-    st.plotly_chart(
-        plot_leaderboard(
-            df,
-            ntop=uid_ntop,
-            group_on=uid_col,
-            agg_col=reward_col,
-            agg=uid_agg
-        )
-    )
-    remove_unsuccessful = st.checkbox('Remove failed completions', value=True)
-    st.plotly_chart(
-        plot_uid_diversty(
-            df,
-            remove_unsuccessful=remove_unsuccessful
+        uids = st.multiselect('UID:', sorted(df_long[f'{uid_src}_uids'].unique()), key='uid')
+        st.markdown('#')
+        st.subheader(f"UID {uid_src.title()} :violet[Weights]")
+
+        plot.weights(
+                df_weights,
+                uids=uids,
         )
-    )
+        
+    with st.expander(f'Show UID **{uid_src}** leaderboard data for **{len(selected_runs)} selected runs**'):
+
+        st.markdown('#')
+        st.subheader(f"UID {uid_src.title()} :violet[Leaderboard]")
+        uid_col1, uid_col2 = st.columns(2)
+        uid_ntop = uid_col1.slider('Number of UIDs:', min_value=1, max_value=50, value=DEFAULT_UID_NTOP, key='uid_ntop')
+        uid_agg = uid_col2.selectbox('Aggregation:', ('mean','min','max','size','nunique'), key='uid_agg')
+
+        plot.leaderboard(
+                df,
+                ntop=uid_ntop,
+                group_on=f'{uid_src}_uids',
+                agg_col=f'{uid_src}_rewards',
+                agg=uid_agg
+            )
+
+
+    with st.expander(f'Show UID **{uid_src}** diversity data for **{len(selected_runs)} selected runs**'):
+
+        st.markdown('#')
+        st.subheader(f"UID {uid_src.title()} :violet[Diversity]")
+        rm_failed = st.checkbox(f'Remove failed **{uid_src}** completions', value=True)
+        plot.uid_diversty(df, rm_failed)
 
 
-completion_ntop_default = 10
+### Completions ###
 with tab3:
 
     st.markdown('#')
     st.subheader('Completion :violet[Leaderboard]')
-    completion_src = default_src
+    completion_info = st.empty()
 
     msg_col1, msg_col2 = st.columns(2)
     completion_src = msg_col1.radio('Select one:', ['followup', 'answer'], horizontal=True, key='completion_src')
-    completion_ntop = msg_col2.slider('Top k:', min_value=1, max_value=50, value=completion_ntop_default, key='completion_ntop')
+    completion_info.info(f"Showing **{completion_src}** completions for **{len(selected_runs)} selected runs**")
+    
+    completion_ntop = msg_col2.slider('Top k:', min_value=1, max_value=50, value=DEFAULT_COMPLETION_NTOP, key='completion_ntop')
 
     completion_col = f'{completion_src}_completions'
     reward_col = f'{completion_src}_rewards'
     uid_col = f'{completion_src}_uids'
 
-    completions = get_completions(df_long, completion_col)
+    completions = inspect.completions(df_long, completion_col)
 
-    # completion_sel = st.radio('Select input method:', ['ntop', 'select','regex'], horizontal=True, key='completion_sel')
     # Get completions with highest average rewards
-    st.plotly_chart(
-        plot_leaderboard(
-            df,
-            ntop=completion_ntop,
-            group_on=completion_col,
-            agg_col=reward_col,
-            agg='mean',
-            alias=True
-        )
+    plot.leaderboard(
+        df,
+        ntop=completion_ntop,
+        group_on=completion_col,
+        agg_col=reward_col,
+        agg='mean',
+        alias=True
     )
-    st.markdown('#')
-    st.subheader('Completion :violet[Rewards]')
 
-    completion_select = st.multiselect('Completions:', completions.index, default=completions.index[:3].tolist())
-    # completion_regex = st.text_input('Completion regex:', value='', key='completion_regex')
+    with st.expander(f'Show **{completion_src}** completion rewards data for **{len(selected_runs)} selected runs**'):
+
+        st.markdown('#')
+        st.subheader('Completion :violet[Rewards]')
 
-    st.plotly_chart(
-        plot_completion_rewards(
+        completion_select = st.multiselect('Completions:', completions.index, default=completions.index[:3].tolist())
+        # completion_regex = st.text_input('Completion regex:', value='', key='completion_regex')
+
+        plot.completion_rewards(
             df,
             completion_col=completion_col,
             reward_col=reward_col,
@@ -306,12 +164,12 @@ with tab3:
             ntop=completion_ntop,
             completions=completion_select,
         )
-    )
 
-with tab4:
-    st.subheader(':pink[Prompt-based scoring]')
-    prompt_src = st.radio('Select one:', ['followup', 'answer'], key='prompt')
 
+### Prompt-based scoring ###
+with tab4:
+    # coming soon
+    st.info('Prompt-based scoring coming soon')
 
     # st.dataframe(df_long_long.filter(regex=prompt_src).head())
 

opendashboards/hello_world.py

@@ -1,3 +0,0 @@
-import streamlit as st
-from utils.utils import test
-st.write('HELLO BOITCHES')

opendashboards/utils/plotting.py

@@ -53,7 +53,7 @@ def plot_weights(scores: pd.DataFrame, ntop: int = 20, uids: List[Union[str, int
     """
 
     # Select subset of columns for plotting
-    if uids is None:
+    if not uids:
         uids = scores.columns[:ntop]
         print(f"Using first {ntop} uids for plotting: {uids}")
 
@@ -249,6 +249,7 @@ def plot_leaderboard(
     else:
         index = rankings.index.astype(str)
 
+    print(f"Using top {ntop} {group_on} by {agg_col}: \n{rankings}")
     return px.bar(
         x=rankings,
         y=index,

opendashboards/utils/utils.py

@@ -24,10 +24,6 @@ from pandas.api.types import is_list_like
 from typing import List, Dict, Any, Union
 
 
-def test():
-    print ("Hello World!")
-
-
 def get_runs(project: str = "openvalidators", filters: Dict[str, Any] = None, return_paths: bool = False) -> List:
     """Download runs from wandb.
 
@@ -78,6 +74,7 @@ def download_data(run_path: Union[str, List] = None, timeout: float = 600) -> pd
         pbar.set_postfix({"total_events": total_events})
 
     df = pd.concat(frames)
+
     # Convert timestamp to datetime.
     df._timestamp = pd.to_datetime(df._timestamp, unit="s")
     df.sort_values("_timestamp", inplace=True)
@@ -95,7 +92,7 @@ def load_data(path: str, nrows: int = None):
     list_cols = [c for c in df.columns if df[c].dtype == "object" and df[c].str.startswith("[").all()]
     # convert string representation of list to list
     df[list_cols] = df[list_cols].applymap(eval, na_action='ignore')
-
+    
     return df
 
 
@@ -128,4 +125,4 @@ def get_list_col_lengths(df: pd.DataFrame) -> Dict[str, int]:
     if len(varying_lengths) > 0:
         print(f"The following columns have varying lengths: {varying_lengths}")
 
-    return {c: v[0] for c, v in list_col_lengths.items()}
+    return {c: v[0] for c, v in list_col_lengths.items() if v}

requirements.txt

@@ -1,3 +1,5 @@
+streamlit==1.23.1
+streamlit-plotly-events==0.0.6
 wandb==0.15.3
 datasets==2.12.0
 plotly==5.14.1

setup.py

@@ -0,0 +1,93 @@
+# The MIT License (MIT)
+# Copyright © 2021 Yuma Rao
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+import pathlib
+import pkg_resources
+from setuptools import setup
+
+
+def read(fname):
+    this_directory = pathlib.Path(__file__).parent
+    long_description = (this_directory / fname).read_text()
+    return long_description
+
+
+def read_requirements(path):
+    with pathlib.Path(path).open() as requirements_txt:
+        return [str(requirement) for requirement in pkg_resources.parse_requirements(requirements_txt)]
+
+
+def get_version(rel_path):
+    for line in read(rel_path).splitlines():
+        if line.startswith("__version__"):
+            delim = '"' if '"' in line else "'"
+            return line.split(delim)[1]
+    else:
+        raise RuntimeError("Unable to find version string.")
+
+
+requirements = read_requirements("requirements.txt")
+
+
+setup(
+    name="opendashboards",
+    version=get_version("opendashboards/__init__.py"),
+    description="Opendashboards is a collection of open source validators for the Bittensor Network.",
+    url="https://github.com/opentensor/opendashboards",
+    author="bittensor.com",
+    packages=["opendashboards"],
+    include_package_data=True,
+    author_email="",
+    license="MIT",
+    long_description=read("README.md"),
+    long_description_content_type="text/markdown",
+    # entry_points={
+    #     "console_scripts": ["foundation-validator = openvalidators.neuron:main"],
+    # },
+    install_requires=requirements,
+    python_requires=">=3.8",
+    classifiers=[
+        "Intended Audience :: Developers",
+        "Topic :: Software Development :: Build Tools",
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Topic :: Scientific/Engineering",
+        "Topic :: Scientific/Engineering :: Mathematics",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+        "Topic :: Software Development",
+        "Topic :: Software Development :: Libraries",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+    ],
+    maintainer="",
+    maintainer_email="",
+    keywords=[
+        "bittensor",
+        "validator",
+        "dashboard",
+        "analytics",
+        "ai",
+        "machine-learning",
+        "deep-learning",
+        "blockchain",
+        "pytorch",
+        "torch",
+        "neural-networks",
+        "cryptocurrency",
+    ],
+)