Adapt syntax for validator v1.1.0 schema

dashboard.py

@@ -9,8 +9,7 @@ from opendashboards.assets import io, inspect, metric, plot
 # Hotkey churn
 
 DEFAULT_PROJECT = "openvalidators"
-DEFAULT_FILTERS = {"tags": {"$in": [f'1.0.{i}' for i in range(10)]}}
-DEFAULT_SELECTED_RUNS = ['kt9bzxii']
+DEFAULT_FILTERS = {"tags": {"$in": [f'1.1.{i}' for i in range(10)]}}
 DEFAULT_SELECTED_HOTKEYS = None
 DEFAULT_SRC = 'followup'
 DEFAULT_COMPLETION_NTOP = 10
@@ -66,13 +65,13 @@ with tab1:
     if n_runs:
         df = io.load_data(df_runs_subset, load=True, save=True)
         df = inspect.clean_data(df)
+        print(f'\nNans in columns: {df.isna().sum()}')
         df_long = inspect.explode_data(df)
-        df_weights = inspect.weights(df)
     else:
         st.info(f'You must select at least one run to load data')
         st.stop()
 
-    metric.runs(df_long, n_runs)
+    metric.runs(df_long)
 
     st.markdown('#')
     st.subheader(":violet[Event] Data")
@@ -83,7 +82,7 @@ with tab1:
         st.dataframe(df_long.head(num_rows) if use_long_checkbox else df.head(num_rows),
                      use_container_width=True)
 
-
+step_types = ['all']+['augment','followup','answer']#list(df.name.unique())
 
 ### UID Health ###
 # TODO: Live time - time elapsed since moving_averaged_score for selected UID was 0 (lower bound so use >Time)
@@ -94,20 +93,21 @@ with tab2:
     st.subheader("UID :violet[Health]")
     st.info(f"Showing UID health metrics for **{n_runs} selected runs**")
 
-    uid_src = st.radio('Select one:', ['followup', 'answer'], horizontal=True, key='uid_src')
-
-    metric.uids(df_long, uid_src)
-    uids = st.multiselect('UID:', sorted(df_long[f'{uid_src}_uids'].unique()), key='uid')
+    uid_src = st.radio('Select event type:', step_types, horizontal=True, key='uid_src')
+    df_uid = df_long[df_long.name.str.contains(uid_src)] if uid_src != 'all' else df_long
+        
+    metric.uids(df_uid, uid_src)
+    uids = st.multiselect('UID:', sorted(df_uid['uids'].unique()), key='uid')
     with st.expander(f'Show UID health data for **{n_runs} selected runs** and **{len(uids)} selected UIDs**'):
         st.markdown('#')
         st.subheader(f"UID {uid_src.title()} :violet[Health]")
         agg_uid_checkbox = st.checkbox('Aggregate UIDs', value=True)
         if agg_uid_checkbox:
-            metric.uids(df_long, uid_src, uids)
+            metric.uids(df_uid, uid_src, uids)
         else:
             for uid in uids:
                 st.caption(f'UID: {uid}')
-                metric.uids(df_long, uid_src, [uid])
+                metric.uids(df_uid, uid_src, [uid])
 
         st.subheader(f'Cumulative completion frequency')
 
@@ -117,18 +117,8 @@ with tab2:
         freq_cumulative = freq_col2.checkbox('Cumulative', value=False, key='freq_cumulative')
         freq_normalize = freq_col2.checkbox('Normalize', value=True, key='freq_normalize')
 
-        plot.uid_completion_counts(df_long, uids=uids, src=uid_src, ntop=freq_ntop, rm_empty=freq_rm_empty, cumulative=freq_cumulative, normalize=freq_normalize)
-
-
-    with st.expander(f'Show UID weights data for **{n_runs} selected runs** and **{len(uids)} selected UIDs**'):
+        plot.uid_completion_counts(df_uid, uids=uids, src=uid_src, ntop=freq_ntop, rm_empty=freq_rm_empty, cumulative=freq_cumulative, normalize=freq_normalize)
 
-        st.markdown('#')
-        st.subheader(f"UID {uid_src.title()} :violet[Weights]")
-
-        plot.weights(
-                df_weights,
-                uids=uids,
-        )
 
     with st.expander(f'Show UID **{uid_src}** leaderboard data for **{n_runs} selected runs**'):
 
@@ -139,10 +129,10 @@ with tab2:
         uid_agg = uid_col2.selectbox('Aggregation:', ('mean','min','max','size','nunique'), key='uid_agg')
 
         plot.leaderboard(
-                df,
+                df_uid,
                 ntop=uid_ntop,
-                group_on=f'{uid_src}_uids',
-                agg_col=f'{uid_src}_rewards',
+                group_on='uids',
+                agg_col='rewards',
                 agg=uid_agg
             )
 
@@ -163,24 +153,22 @@ with tab3:
     completion_info = st.empty()
 
     msg_col1, msg_col2 = st.columns(2)
-    completion_src = msg_col1.radio('Select one:', ['followup', 'answer'], horizontal=True, key='completion_src')
+    # completion_src = msg_col1.radio('Select one:', ['followup', 'answer'], horizontal=True, key='completion_src')
+    completion_src = st.radio('Select event type:', step_types, horizontal=True, key='completion_src')
+    df_comp = df_long[df_long.name==completion_src] if completion_src != 'all' else df_long
+    
     completion_info.info(f"Showing **{completion_src}** completions for **{n_runs} selected runs**")
 
     completion_ntop = msg_col2.slider('Top k:', min_value=1, max_value=50, value=DEFAULT_COMPLETION_NTOP, key='completion_ntop')
 
-    completion_col = f'{completion_src}_completions'
-    reward_col = f'{completion_src}_rewards'
-    uid_col = f'{completion_src}_uids'
-    time_col = f'{completion_src}_times'
-
-    completions = inspect.completions(df_long, completion_col)
+    completions = inspect.completions(df_long, 'completions')
 
     # Get completions with highest average rewards
     plot.leaderboard(
         df,
         ntop=completion_ntop,
-        group_on=completion_col,
-        agg_col=reward_col,
+        group_on='completions',
+        agg_col='rewards',
         agg='mean',
         alias=True
     )
@@ -195,9 +183,9 @@ with tab3:
 
         plot.completion_rewards(
             df,
-            completion_col=completion_col,
-            reward_col=reward_col,
-            uid_col=uid_col,
+            completion_col='completions',
+            reward_col='rewards',
+            uid_col='uids',
             ntop=completion_ntop,
             completions=completion_select,
         )
@@ -209,14 +197,14 @@ with tab3:
         st.markdown('#')
         st.subheader('Completion :violet[Length]')
 
-        words_checkbox = st.checkbox('Use words', value=True, key='words_checkbox')
+        completion_length_radio = st.radio('Use: ', ['characters','words','sentences'], key='completion_length_radio')
 
         plot.completion_length_time(
             df,
-            completion_col=completion_col,
-            uid_col=uid_col,
-            time_col=time_col,
-            words=words_checkbox,
+            completion_col='completions',
+            uid_col='uids',
+            time_col='completion_times',
+            length_opt=completion_length_radio,
         )
 
 ### Prompt-based scoring ###

opendashboards/assets/inspect.py

@@ -4,7 +4,7 @@ import pandas as pd
 import opendashboards.utils.utils as utils
 
 def clean_data(df):
-    return df.dropna(subset=df.filter(regex='completions|rewards').columns, how='all')
+    return df.dropna(subset=df.filter(regex='completions|rewards').columns, how='any')
 
 @st.cache_data
 def explode_data(df):
@@ -22,16 +22,6 @@ def explode_data(df):
 def completions(df_long, col):
     return df_long[col].value_counts()
 
-@st.cache_data
-def weights(df, index='_timestamp'):
-    # Create a column for each UID and show most recent rows
-    scores = df['moving_averaged_scores'].apply(pd.Series).fillna(method='ffill')
-    if index in df.columns:
-        scores.index = df[index]
-
-    # rename columns
-    scores.rename({i: f'UID-{i}' for i in range(scores.shape[1])}, axis=1, inplace=True)
-    return scores
 
 def run_event_data(df_runs, df, selected_runs):
 

opendashboards/assets/io.py

@@ -25,7 +25,7 @@ def load_runs(project, filters, min_steps=10):
     for i, run in enumerate(all_runs):
         
         summary = run.summary
-        step = summary.get('_step',0)
+        step = summary.get('_step',-1) + 1
         if step < min_steps:
             msg.warning(f'Skipped run `{run.name}` because it contains {step} events (<{min_steps})')
             continue
@@ -46,11 +46,11 @@ def load_runs(project, filters, min_steps=10):
             'num_steps': step,
             'num_completions': step*sum(len(v) for k, v in run.summary.items() if k.endswith('completions') and isinstance(v, list)),
             'entity': run.entity,
-            'id': run.id,
-            'name': run.name,
+            'run_id': run.id,
+            'run_name': run.name,
             'project': run.project,
             'url': run.url,
-            'path': os.path.join(run.entity, run.project, run.id),
+            'run_path': os.path.join(run.entity, run.project, run.id),
             'start_time': pd.to_datetime(end_time-duration, unit="s"),
             'end_time': pd.to_datetime(end_time, unit="s"),
             'duration': pd.to_timedelta(duration, unit="s").round('s'),
@@ -78,7 +78,7 @@ def load_data(selected_runs, load=True, save=False):
         run = selected_runs.loc[idx]
         prog_msg = f'Loading data {i/len(selected_runs)*100:.0f}% ({successful}/{len(selected_runs)} runs, {n_events} events)'
 
-        file_path = os.path.join('data',f'history-{run.id}.csv')
+        file_path = os.path.join('data',f'history-{run.run_id}.csv')
 
         if load and os.path.exists(file_path):
             progress.progress(i/len(selected_runs),f'{prog_msg}... **reading** `{file_path}`')
@@ -89,18 +89,19 @@ def load_data(selected_runs, load=True, save=False):
                 st.exception(e)
                 continue
         else:
-            progress.progress(i/len(selected_runs),f'{prog_msg}... **downloading** `{run.path}`')
+            progress.progress(i/len(selected_runs),f'{prog_msg}... **downloading** `{run.run_path}`')
             try:
-                # Download the history from wandb
-                df = utils.download_data(run.path)
-                # Add metadata to the dataframe
-                df.assign(**run.to_dict())
+                # Download the history from wandb and add metadata
+                df = utils.download_data(run.run_path).assign(**run.to_dict())
 
+                print(f'Downloaded {df.shape[0]} events from `{run.run_path}`. Columns: {df.columns}')
+                df.info()
+                
                 if save and run.state != 'running':
                     df.to_csv(file_path, index=False)
                     # st.info(f'Saved history to {file_path}')
             except Exception as e:
-                info.warning(f'Failed to download history for `{run.path}`')
+                info.warning(f'Failed to download history for `{run.run_path}`')
                 st.exception(e)
                 continue
 

opendashboards/assets/metric.py

@@ -33,15 +33,26 @@ def wandb(df_runs):
 
 
 @st.cache_data
-def runs(df_long, n_runs):
-
-    col1, col2, col3 = st.columns(3)
-    col1.metric(label="Runs", value=n_runs)
-    col1.metric(label="Events", value=df_long.shape[0])
-    col2.metric(label="Followup UIDs", value=df_long.followup_uids.nunique())
-    col2.metric(label="Answer UIDs", value=df_long.answer_uids.nunique())
-    col3.metric(label="Unique Followups", value=df_long.followup_completions.nunique())
-    col3.metric(label="Unique Answers", value=df_long.answer_completions.nunique())
+def runs(df_long):
+    
+    col1, col2, col3, col4 = st.columns(4)
+    print(df_long.columns)
+
+    # Convert to appropriate units e.g. 1.2k instead of 1200.c
+    col1.metric('Runs', fmt(df_long.run_id.nunique()))
+    col2.metric('Hotkeys', fmt(df_long.hotkey.nunique()))
+    col3.metric('Events', fmt(df_long.groupby(['run_id','_step']).ngroups))
+    col4.metric('Completions', fmt(df_long.shape[0]))
+    
+    name_type = df_long.name.apply(lambda x: x if not x[-1].isdigit() else x[:-1])
+    aggs = df_long.groupby(name_type).agg({'uids': 'nunique', 'completions': 'nunique'})
+    print(aggs)
+    for i,c in enumerate(st.columns(len(aggs))):
+        name = aggs.index[i].title()
+        uid_unique, comp_unique = aggs.iloc[i]
+        c.metric(label=f'{name} UIDs', value=uid_unique) 
+        c.metric(label=f'{name} Completions', value=comp_unique)
+
     st.markdown('----')
 
 
@@ -49,30 +60,28 @@ def runs(df_long, n_runs):
 @st.cache_data
 def uids(df_long, src, uids=None):
 
-    uid_col = f'{src}_uids'
-    completion_col = f'{src}_completions'
     nsfw_col = f'{src}_nsfw_scores'
-    reward_col = f'{src}_rewards'
 
     if uids:
-        df_long = df_long.loc[df_long[uid_col].isin(uids)]
+        df_long = df_long.loc[df_long['uids'].isin(uids)]
 
     col1, col2, col3, col4 = st.columns(4)
     col1.metric(
         label="Success %",
-        value=f'{df_long.loc[df_long[completion_col].str.len() > 0].shape[0]/df_long.shape[0] * 100:.1f}',
+        value=f'{df_long.loc[df_long["completions"].str.len() > 0].shape[0]/df_long.shape[0] * 100:.1f}',
         help='Number of successful completions divided by total number of events'
     )
     col2.metric(
         label="Diversity %",
-        value=f'{df_long[completion_col].nunique()/df_long.shape[0] * 100:.1f}',
+        value=f'{df_long["completions"].nunique()/df_long.shape[0] * 100:.1f}',
         help='Number of unique completions divided by total number of events'
     )
     # uniqueness can be expressed as the average number of unique completions per uid divided by all unique completions
+    # uniqueness is the shared completions between selected uids 
 
     col3.metric(
         label="Uniqueness %",
-        value=f'{df_long.groupby(uid_col)[completion_col].nunique().mean()/df_long[completion_col].nunique() * 100:.1f}',
+        value=f'{df_long.groupby("uids")["completions"].nunique().mean()/df_long["completions"].nunique() * 100:.1f}',
         help='Average number of unique completions per uid divided by all unique completions'
     )
     col4.metric(

opendashboards/assets/plot.py

@@ -55,14 +55,14 @@ def weights(df, uids, ntop=10):
         use_container_width=True
     )
 
-def completion_length_time(df, completion_col, uid_col, time_col, words=False):
+def completion_length_time(df, completion_col, uid_col, time_col, length_opt='characters'):
     return st.plotly_chart(
         plotting.plot_completion_length_time(
             df,
             uid_col=uid_col,
             completion_col=completion_col,
             time_col=time_col,
-            words=words
+            length_opt=length_opt
         ),
         use_container_width=True
     )

opendashboards/utils/plotting.py

@@ -62,16 +62,16 @@ def plot_weights(scores: pd.DataFrame, ntop: int = 20, uids: List[Union[str, int
     ).update_traces(opacity=0.7)
 
 
-def plot_uid_diversty(df: pd.DataFrame, remove_unsuccessful: bool = False) -> go.Figure:
+def plot_uid_diversty(df: pd.DataFrame, x: str = 'followup', y: str = 'answer', remove_unsuccessful: bool = False) -> go.Figure:
     """Plot uid diversity as measured by ratio of unique to total completions.
 
     Args:
         df (pd.DataFrame): Dataframe of event log.
     """
-    uid_cols = ["followup_uids", "answer_uids"]
-    completion_cols = ["followup_completions", "answer_completions"]
-    reward_cols = ["followup_rewards", "answer_rewards"]
-    list_cols = uid_cols + completion_cols + reward_cols
+    return px.scatter(x=[1,2,3],y=[1,2,3])
+    xrows = df.loc[df.name.str.contains(x)]
+    yrows = df.loc[df.name.str.contains(y)]
+    df = pd.merge(xrows, yrows, on='uid', suffixes=('_followup', '_answer'))
 
     df = df[list_cols].explode(column=list_cols)
     if remove_unsuccessful:
@@ -98,7 +98,7 @@ def plot_uid_diversty(df: pd.DataFrame, remove_unsuccessful: bool = False) -> go
         x="diversity_followup",
         y="diversity_answer",
         opacity=0.35,
-        # size="followup_completions_size",
+        # size="completions_size",
         color="reward_mean",
         hover_data=["UID"] + merged.columns.tolist(),
         marginal_x="histogram",
@@ -112,7 +112,7 @@ def plot_uid_diversty(df: pd.DataFrame, remove_unsuccessful: bool = False) -> go
 
 def plot_completion_rates(
     df: pd.DataFrame,
-    msg_col: str = "all_completions",
+    msg_col: str = "completions",
     time_interval: str = "H",
     time_col: str = "_timestamp",
     ntop: int = 20,
@@ -123,7 +123,7 @@ def plot_completion_rates(
 
     Args:
         df (pd.DataFrame): Dataframe of event log.
-        msg_col (str, optional): List-like column containing completions. Defaults to 'all_completions'.
+        msg_col (str, optional): List-like column containing completions. Defaults to 'completions'.
         time_interval (str, optional): Pandas time interval. Defaults to 'H'. See https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#timeseries-offset-aliases
         time_col (str, optional): Column containing timestamps as pd.Datetime. Defaults to '_timestamp'.
         ntop (int, optional): Number of completions to plot. Defaults to 20.
@@ -163,10 +163,10 @@ def plot_completion_rates(
 
 def plot_completion_rewards(
     df: pd.DataFrame,
-    msg_col: str = "followup_completions",
-    reward_col: str = "followup_rewards",
+    msg_col: str = "completions",
+    reward_col: str = "rewards",
     time_col: str = "_timestamp",
-    uid_col: str = "followup_uids",
+    uid_col: str = "uids",
     ntop: int = 3,
     completions: List[str] = None,
     completion_regex: str = None,
@@ -175,9 +175,10 @@ def plot_completion_rewards(
 
     Args:
         df (pd.DataFrame): Dataframe of event log.
-        msg_col (str, optional): List-like column containing completions. Defaults to 'followup_completions'.
-        reward_col (str, optional): List-like column containing rewards. Defaults to 'followup_rewards'.
+        msg_col (str, optional): List-like column containing completions. Defaults to 'completions'.
+        reward_col (str, optional): List-like column containing rewards. Defaults to 'rewards'.
         time_col (str, optional): Column containing timestamps as pd.Datetime. Defaults to '_timestamp'.
+        uid_col (str, optional): Column containing UIDs. Defaults to 'uids'.
         ntop (int, optional): Number of completions to plot. Defaults to 20.
         completions (List[str], optional): List of completions to plot. Defaults to None.
         completion_regex (str, optional): Regex to match completions. Defaults to None.
@@ -229,8 +230,8 @@ def plot_completion_rewards(
 
 def plot_leaderboard(
     df: pd.DataFrame,
-    group_on: str = "answer_uids",
-    agg_col: str = "answer_rewards",
+    group_on: str = "uids",
+    agg_col: str = "rewards",
     agg: str = "mean",
     ntop: int = 10,
     alias: bool = False,
@@ -239,8 +240,8 @@ def plot_leaderboard(
 
     Args:
         df (pd.DataFrame): Dataframe of event log.
-        group_on (str, optional): Entities to use for grouping. Defaults to 'answer_uids'.
-        agg_col (str, optional): Column to aggregate. Defaults to 'answer_rewards'.
+        group_on (str, optional): Entities to use for grouping. Defaults to 'uids'.
+        agg_col (str, optional): Column to aggregate. Defaults to 'rewards'.
         agg (str, optional): Aggregation function. Defaults to 'mean'.
         ntop (int, optional): Number of entities to plot. Defaults to 10.
         alias (bool, optional): Whether to use aliases for indices. Defaults to False.
@@ -269,14 +270,14 @@ def plot_leaderboard(
 
 
 def plot_dendrite_rates(
-    df: pd.DataFrame, uid_col: str = "answer_uids", reward_col: str = "answer_rewards", ntop: int = 20, uids: List[int] = None
+    df: pd.DataFrame, uid_col: str = "uids", reward_col: str = "rewards", ntop: int = 20, uids: List[int] = None
 ) -> go.Figure:
     """Makes a bar chart of the success rate of dendrite calls for a given set of uids.
 
     Args:
         df (pd.DataFrame): Dataframe of event log.
-        uid_col (str, optional): Column containing uids. Defaults to 'answer_uids'.
-        reward_col (str, optional): Column containing rewards. Defaults to 'answer_rewards'.
+        uid_col (str, optional): Column containing uids. Defaults to 'uids'.
+        reward_col (str, optional): Column containing rewards. Defaults to 'rewards'.
         ntop (int, optional): Number of uids to plot. Defaults to 20.
         uids (List[int], optional): List of uids to plot. Defaults to None.
 
@@ -307,30 +308,36 @@ def plot_dendrite_rates(
 
 def plot_completion_length_time(
     df: pd.DataFrame,
-    uid_col: str = "answer_uids",
-    completion_col: str = "answer_completions",
-    time_col: str = "answer_times",
+    uid_col: str = "uids",
+    completion_col: str = "completions",
+    time_col: str = "completion_times",
     uids: List[int] = None,
-    words: bool = False,
+    length_opt: str = 'characters',
 ) -> go.Figure:
 
+
     df = df[[uid_col, completion_col, time_col]].explode(column=[uid_col, completion_col, time_col])
     df["time"] = df[time_col].astype(float)
     if uids is not None:
         df = df.loc[df[uid_col].isin(uids)]
         
-    if words:
+        
+    if length_opt == 'characters':
+        df["completion_length"] = df[completion_col].str.len()
+    elif length_opt == 'words':
         df["completion_length"] = df[completion_col].str.split().str.len()
+    elif length_opt == 'sentences':
+        df["completion_length"] = df[completion_col].str.split('.').str.len()
     else:
-        df["completion_length"] = df[completion_col].str.len()
+        raise ValueError(f"length_opt must be one of 'words', 'characters', or 'sentences', got {length_opt}")
 
     return px.scatter(
         df,
         x='completion_length',
         y='time',
         color=uid_col if uids is not None else None,
-        labels={"completion_length": f"Completion Length, {'Words' if words else 'Characters'}", "time": "Time (s)"},
-        title=f"Completion Length vs Time, {'Words' if words else 'Characters'}",
+        labels={"completion_length": f"Completion Length, {length_opt.title()}", "time": "Time (s)"},
+        title=f"Completion Length vs Time, {length_opt.title()}",
         marginal_x="histogram",
         marginal_y="histogram",
         hover_data=[uid_col, completion_col],
@@ -348,8 +355,8 @@ def plot_uid_completion_counts(
     normalize: bool = True,
 ) -> go.Figure:
 
-    completion_col = f'{src}_completions'
-    uid_col = f'{src}_uids'
+    completion_col = f'completions'
+    uid_col = f'uids'
     if rm_empty:
         df = df.loc[df[completion_col].str.len()>0]
 
@@ -378,8 +385,8 @@ def plot_uid_completion_counts(
 
 def plot_network_embedding(
     df: pd.DataFrame,
-    uid_col: str = "followup_uids",
-    completion_col: str = "followup_completions",
+    uid_col: str = "uids",
+    completion_col: str = "completions",
     ntop: int = 1,
     uids: List[int] = None,
 ) -> go.Figure:
@@ -388,8 +395,8 @@ def plot_network_embedding(
     Args:
         df (pd.DataFrame): Dataframe of event log.
 
-        uid_col (str, optional): Column containing uids. Defaults to 'answer_uids'.
-        completion_col (str, optional): Column containing completions. Defaults to 'followup_completions'.
+        uid_col (str, optional): Column containing uids. Defaults to 'uids'.
+        completion_col (str, optional): Column containing completions. Defaults to 'completions'.
         ntop (int, optional): Number of uids to plot. Defaults to 20.
         hover_data (List[str], optional): Columns to include in hover data. Defaults to None.
         uids (List[int], optional): List of uids to plot. Defaults to None.