HELLO

dashboard.py

@@ -0,0 +1,317 @@
+import streamlit as st
+import pandas as pd
+import os
+import re
+import time
+from .utils import get_runs, download_data, get_list_col_lengths, explode_data
+import .plotting as plotting
+
+
+# dendrite time versus completion length
+# prompt-based completion score stats
+
+
+DEFAULT_PROJECT = "openvalidators"
+DEFAULT_FILTERS = {"tags": {"$in": ["1.0.0", "1.0.1", "1.0.2", "1.0.3", "1.0.4"]}}
+
+@st.cache_data
+def load_runs(project=DEFAULT_PROJECT, filters=DEFAULT_FILTERS, min_steps=10):
+    runs = []
+    msg = st.empty()
+    for run in get_runs(project, filters):
+        step = run.summary.get('_step',0)
+        if step < min_steps:
+            msg.warning(f'Skipped run `{run.name}` because it contains {step} events (<{min_steps})')
+            continue
+
+        duration = run.summary.get('_runtime')
+        end_time = run.summary.get('_timestamp')
+        # extract values for selected tags
+        rules = {'hotkey': re.compile('^[0-9a-z]{48}$',re.IGNORECASE), 'version': re.compile('^\\d\.\\d+\.\\d+$'), 'spec_version': re.compile('\\d{4}$')}
+        # tags = {k: tag for k,tag in zip(('hotkey','version','spec_version'),run.tags)}
+        tags = {k: tag for k, rule in rules.items() for tag in run.tags if rule.match(tag)}
+        # include bool flag for remaining tags
+        tags.update({k: k in run.tags for k in ('mock','custom_gating_model','nsfw_filter','outsource_scoring','disable_set_weights')})
+
+        runs.append({
+            'state': run.state,
+            'num_steps': step,
+            'entity': run.entity,
+            'id': run.id,
+            'name': run.name,
+            'project': run.project,
+            'url': run.url,
+            'path': os.path.join(run.entity, run.project, run.id),
+            'start_time': pd.to_datetime(end_time-duration, unit="s"),
+            'end_time': pd.to_datetime(end_time, unit="s"),
+            'duration': pd.to_datetime(duration, unit="s"),
+            # 'tags': run.tags,
+            **tags
+        })
+    msg.empty()
+    return pd.DataFrame(runs).astype({'state': 'category', 'hotkey': 'category', 'version': 'category', 'spec_version': 'category'})
+
+
+@st.cache_data
+def load_data(selected_runs, load=True, save=False):
+
+    frames = []
+    n_events = 0
+    progress = st.progress(0, 'Loading data')
+    for i, idx in enumerate(selected_runs.index):
+        run = selected_runs.loc[idx]
+        prog_msg = f'Loading data {i/len(selected_runs)*100:.0f}% ({i}/{len(selected_runs)} runs, {n_events} events)'
+
+        file_path = f'data/history-{run.id}.csv'
+
+        if load and os.path.exists(file_path):
+            progress.progress(i/len(selected_runs),f'{prog_msg}... reading {file_path}')
+            df = pd.read_csv(file_path)
+            # filter out events with missing step length
+            df = df.loc[df.step_length.notna()]
+
+            # detect list columns which as stored as strings
+            list_cols = [c for c in df.columns if df[c].dtype == "object" and df[c].str.startswith("[").all()]
+            # convert string representation of list to list
+            df[list_cols] = df[list_cols].applymap(eval, na_action='ignore')
+
+        else:
+            try:
+                # Download the history from wandb
+                progress.progress(i/len(selected_runs),f'{prog_msg}... downloading `{run.path}`')
+                df = download_data(run.path)
+                df.assign(**run.to_dict())
+                if not os.path.exists('data/'):
+                    os.makedirs(file_path)
+
+                if save and run.state != 'running':
+                    df.to_csv(file_path, index=False)
+                    # st.info(f'Saved history to {file_path}')
+            except Exception as e:
+                st.error(f'Failed to download history for `{run.path}`')
+                st.exception(e)
+                continue
+
+        frames.append(df)
+        n_events += df.shape[0]
+
+    progress.empty()
+    # Remove rows which contain chain weights as it messes up schema
+    return pd.concat(frames)
+
+@st.cache_data
+def get_exploded_data(df):
+    list_cols = get_list_col_lengths(df)
+    return explode_data(df, list(list_cols))
+
+@st.cache_data
+def get_completions(df_long, col):
+    return df_long[col].value_counts()
+
+@st.cache_data
+def plot_uid_diversty(df, remove_unsuccessful=True):
+    return plotting.plot_uid_diversty(df, remove_unsuccessful=remove_unsuccessful)
+
+@st.cache_data
+def plot_leaderboard(df, ntop, group_on, agg_col, agg, alias=False):
+    return plotting.plot_leaderboard(df, ntop=ntop, group_on=group_on, agg_col=agg_col, agg=agg, alias=alias)
+
+@st.cache_data
+def plot_completion_rewards(df, completion_col, reward_col, uid_col, ntop, completions=None, completion_regex=None):
+    return plotting.plot_completion_rewards(df, msg_col=completion_col, reward_col=reward_col, uid_col=uid_col, ntop=ntop, completions=completions, completion_regex=completion_regex)
+
+@st.cache_data
+def uid_metrics(df_long, src, uid=None):
+
+    uid_col = f'{src}_uids'
+    completion_col = f'{src}_completions'
+    nsfw_col = f'{src}_nsfw_scores'
+    reward_col = f'{src}_rewards'
+
+    if uid is not None:
+        df_long = df_long.loc[df_long[uid_col] == uid]
+
+    col1, col2, col3 = st.columns(3)
+    col1.metric(
+        label="Success %",
+        value=f'{df_long.loc[df_long[completion_col].str.len() > 0].shape[0]/df_long.shape[0] * 100:.1f}'
+    )
+    col2.metric(
+        label="Diversity %",
+        value=f'{df_long[completion_col].nunique()/df_long.shape[0] * 100:.1f}'
+    )
+    col3.metric(
+        label="Toxicity %",
+        value=f'{df_long[nsfw_col].mean() * 100:.1f}' if nsfw_col in df_long.columns else 'N/A'
+    )
+
+st.title('Validator :red[Analysis] Dashboard :eyes:')
+# add vertical space
+st.markdown('#')
+st.markdown('#')
+
+
+with st.sidebar:
+    st.sidebar.header('Pages')
+
+with st.spinner(text=f'Checking wandb...'):
+    df_runs = load_runs()
+# get rows where start time is older than 24h ago
+df_runs_old = df_runs.loc[df_runs.start_time < pd.to_datetime(time.time()-24*60*60, unit='s')]
+
+col1, col2, col3 = st.columns(3)
+
+col1.metric('Runs', df_runs.shape[0], delta=f'{df_runs.shape[0]-df_runs_old.shape[0]} (24h)')
+col2.metric('Hotkeys', df_runs.hotkey.nunique(), delta=f'{df_runs.hotkey.nunique()-df_runs_old.hotkey.nunique()} (24h)')
+col3.metric('Events', df_runs.num_steps.sum(), delta=f'{df_runs.num_steps.sum()-df_runs_old.num_steps.sum()} (24h)')
+
+# https://wandb.ai/opentensor-dev/openvalidators/runs/kt9bzxii/overview?workspace=
+# all_run_paths = ['opentensor-dev/openvalidators/kt9bzxii'] # pedro long run
+
+run_ids = df_runs.id
+default_selected_runs = ['kt9bzxii']
+selected_runs = default_selected_runs
+
+# add vertical space
+st.markdown('#')
+st.markdown('#')
+
+
+tab1, tab2, tab3, tab4 = st.tabs(["Wandb Runs", "UID Health", "Completions", "Prompt-based scoring"])
+
+# src = st.radio('Choose data source:', ['followup', 'answer'], horizontal=True, key='src')
+# list_list_cols = get_list_col_lengths(df_long)
+# df_long_long = explode_data(df_long, list(list_list_cols))
+
+with tab1:
+
+    st.markdown('#')
+    st.subheader(":violet[Wandb] Runs")
+
+    # Load data
+    df = load_data(df_runs.loc[run_ids.isin(selected_runs)], load=True, save=True)
+    df_long = get_exploded_data(df)
+
+    col1, col2, col3, col4 = st.columns(4)
+    col1.metric(label="Selected runs", value=len(selected_runs))
+    col2.metric(label="Events", value=df.shape[0]) #
+    col3.metric(label="UIDs", value=df_long.followup_uids.nunique())
+    col4.metric(label="Unique completions", value=df_long.followup_uids.nunique())
+
+    selected_runs = st.multiselect(f'Runs ({len(df_runs)})', run_ids, default=selected_runs)
+
+    st.markdown('#')
+    st.subheader("View :violet[Data]")
+
+    show_col1, show_col2 = st.columns(2)
+    show_runs = show_col1.checkbox('Show runs', value=True)
+    show_events = show_col2.checkbox('Show events', value=False)
+    if show_runs:
+        st.markdown(f'Wandb info for **{len(selected_runs)} selected runs**:')
+        st.dataframe(df_runs.loc[run_ids.isin(selected_runs)],
+                    column_config={
+                        "url": st.column_config.LinkColumn("URL"),
+                    }
+        )
+
+    if show_events:
+        st.markdown(f'Raw events for **{len(selected_runs)} selected runs**:')
+        st.dataframe(df.head(50),
+                    column_config={
+                        "url": st.column_config.LinkColumn("URL"),
+                    }
+        )
+
+default_src = 'followup'
+with tab2:
+
+    st.markdown('#')
+    st.subheader("UID :violet[Health]")
+    uid_src = default_src
+
+    # uid = st.selectbox('UID:', sorted(df_long[uid_col].unique()), key='uid')
+
+    uid_metrics(df_long, uid_src)
+    uid_src = st.radio('Select one:', ['followup', 'answer'], horizontal=True, key='uid_src')
+    uid_col = f'{uid_src}_uids'
+    reward_col = f'{uid_src}_rewards'
+
+    st.markdown('#')
+    st.subheader("UID :violet[Leaderboard]")
+    uid_ntop_default = 10
+
+    uid_col1, uid_col2 = st.columns(2)
+    uid_ntop = uid_col1.slider('Number of UIDs:', min_value=1, max_value=50, value=uid_ntop_default, key='uid_ntop')
+    uid_agg = uid_col2.selectbox('Aggregation:', ('mean','min','max','size','nunique'), key='uid_agg')
+
+    st.plotly_chart(
+        plot_leaderboard(
+            df,
+            ntop=uid_ntop,
+            group_on=uid_col,
+            agg_col=reward_col,
+            agg=uid_agg
+        )
+    )
+    remove_unsuccessful = st.checkbox('Remove failed completions', value=True)
+    st.plotly_chart(
+        plot_uid_diversty(
+            df,
+            remove_unsuccessful=remove_unsuccessful
+        )
+    )
+
+
+completion_ntop_default = 10
+with tab3:
+
+    st.markdown('#')
+    st.subheader('Completion :violet[Leaderboard]')
+    completion_src = default_src
+
+    msg_col1, msg_col2 = st.columns(2)
+    completion_src = msg_col1.radio('Select one:', ['followup', 'answer'], horizontal=True, key='completion_src')
+    completion_ntop = msg_col2.slider('Top k:', min_value=1, max_value=50, value=completion_ntop_default, key='completion_ntop')
+
+    completion_col = f'{completion_src}_completions'
+    reward_col = f'{completion_src}_rewards'
+    uid_col = f'{completion_src}_uids'
+
+    completions = get_completions(df_long, completion_col)
+
+    # completion_sel = st.radio('Select input method:', ['ntop', 'select','regex'], horizontal=True, key='completion_sel')
+    # Get completions with highest average rewards
+    st.plotly_chart(
+        plot_leaderboard(
+            df,
+            ntop=completion_ntop,
+            group_on=completion_col,
+            agg_col=reward_col,
+            agg='mean',
+            alias=True
+        )
+    )
+    st.markdown('#')
+    st.subheader('Completion :violet[Rewards]')
+
+    completion_select = st.multiselect('Completions:', completions.index, default=completions.index[:3].tolist())
+    # completion_regex = st.text_input('Completion regex:', value='', key='completion_regex')
+
+    st.plotly_chart(
+        plot_completion_rewards(
+            df,
+            completion_col=completion_col,
+            reward_col=reward_col,
+            uid_col=uid_col,
+            ntop=completion_ntop,
+            completions=completion_select,
+        )
+    )
+
+with tab4:
+    st.subheader(':pink[Prompt-based scoring]')
+    prompt_src = st.radio('Select one:', ['followup', 'answer'], key='prompt')
+
+
+    # st.dataframe(df_long_long.filter(regex=prompt_src).head())
+

hello_world.py

@@ -0,0 +1,11 @@
+import streamlit as st
+import pandas as pd
+import os
+import re
+import time
+from .utils import get_runs, download_data, get_list_col_lengths, explode_data
+import .plotting as plotting
+
+
+
+st.write('HELLO BOITCHES')

plotting.py

@@ -0,0 +1,362 @@
+# The MIT License (MIT)
+# Copyright © 2021 Yuma Rao
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import tqdm
+
+import pandas as pd
+import numpy as np
+import networkx as nx
+
+import plotly.express as px
+import plotly.graph_objects as go
+
+from typing import List, Union
+
+plotly_config = {"width": 800, "height": 600, "template": "plotly_white"}
+
+
+def plot_throughput(df: pd.DataFrame, n_minutes: int = 10) -> go.Figure:
+    """Plot throughput of event log.
+
+    Args:
+        df (pd.DataFrame): Dataframe of event log.
+        n_minutes (int, optional): Number of minutes to aggregate. Defaults to 10.
+    """
+
+    rate = df.resample(rule=f"{n_minutes}T", on="_timestamp").size()
+    return px.line(
+        x=rate.index, y=rate, title="Event Log Throughput", labels={"x": "", "y": f"Logs / {n_minutes} min"}, **plotly_config
+    )
+
+
+def plot_weights(scores: pd.DataFrame, ntop: int = 20, uids: List[Union[str, int]] = None) -> go.Figure:
+    """_summary_
+
+    Args:
+        scores (pd.DataFrame): Dataframe of scores. Should be indexed by timestamp and have one column per uid.
+        ntop (int, optional): Number of uids to plot. Defaults to 20.
+        uids (List[Union[str, int]], optional): List of uids to plot, should match column names. Defaults to None.
+    """
+
+    # Select subset of columns for plotting
+    if uids is None:
+        uids = scores.columns[:ntop]
+        print(f"Using first {ntop} uids for plotting: {uids}")
+
+    return px.line(
+        scores, y=uids, title="Moving Averaged Scores", labels={"_timestamp": "", "value": "Score"}, **plotly_config
+    ).update_traces(opacity=0.7)
+
+
+def plot_uid_diversty(df: pd.DataFrame, remove_unsuccessful: bool = False) -> go.Figure:
+    """Plot uid diversity as measured by ratio of unique to total completions.
+
+    Args:
+        df (pd.DataFrame): Dataframe of event log.
+    """
+    uid_cols = ["followup_uids", "answer_uids"]
+    completion_cols = ["followup_completions", "answer_completions"]
+    reward_cols = ["followup_rewards", "answer_rewards"]
+    list_cols = uid_cols + completion_cols + reward_cols
+
+    df = df[list_cols].explode(column=list_cols)
+    if remove_unsuccessful:
+        # remove unsuccessful completions, as indicated by empty completions
+        for col in completion_cols:
+            df = df[df[col].str.len() > 0]
+
+    frames = []
+    for uid_col, completion_col, reward_col in zip(uid_cols, completion_cols, reward_cols):
+        frame = df.groupby(uid_col).agg({completion_col: ["nunique", "size"], reward_col: "mean"})
+        # flatten multiindex columns
+        frame.columns = ["_".join(col) for col in frame.columns]
+        frame["diversity"] = frame[f"{completion_col}_nunique"] / frame[f"{completion_col}_size"]
+        frames.append(frame)
+
+    merged = pd.merge(*frames, left_index=True, right_index=True, suffixes=("_followup", "_answer"))
+    merged["reward_mean"] = merged.filter(regex="rewards_mean").mean(axis=1)
+
+    merged.index.name = "UID"
+    merged.reset_index(inplace=True)
+
+    return px.scatter(
+        merged,
+        x="diversity_followup",
+        y="diversity_answer",
+        opacity=0.3,
+        size="followup_completions_size",
+        color="reward_mean",
+        hover_data=["UID"] + merged.columns.tolist(),
+        marginal_x="histogram",
+        marginal_y="histogram",
+        color_continuous_scale=px.colors.sequential.Bluered,
+        labels={"x": "Followup diversity", "y": "Answer diversity"},
+        title="Diversity of completions by UID",
+        **plotly_config,
+    )
+
+
+def plot_completion_rates(
+    df: pd.DataFrame,
+    msg_col: str = "all_completions",
+    time_interval: str = "H",
+    time_col: str = "_timestamp",
+    ntop: int = 20,
+    completions: List[str] = None,
+    completion_regex: str = None,
+) -> go.Figure:
+    """Plot completion rates. Useful for identifying common completions and attacks.
+
+    Args:
+        df (pd.DataFrame): Dataframe of event log.
+        msg_col (str, optional): List-like column containing completions. Defaults to 'all_completions'.
+        time_interval (str, optional): Pandas time interval. Defaults to 'H'. See https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#timeseries-offset-aliases
+        time_col (str, optional): Column containing timestamps as pd.Datetime. Defaults to '_timestamp'.
+        ntop (int, optional): Number of completions to plot. Defaults to 20.
+        completions (List[str], optional): List of completions to plot. Defaults to None.
+        completion_regex (str, optional): Regex to match completions. Defaults to None.
+
+    """
+
+    df = df[[time_col, msg_col]].explode(column=msg_col)
+
+    if completions is None:
+        completion_counts = df[msg_col].value_counts()
+        if completion_regex is not None:
+            completions = completion_counts[completion_counts.index.str.contains(completion_regex)].index[:ntop]
+            print(f"Using {len(completions)} completions which match {completion_regex!r}: \n{completions}")
+        else:
+            completions = completion_counts.index[:ntop]
+            print(f"Using top {len(completions)} completions: \n{completions}")
+
+    period = df[time_col].dt.to_period(time_interval)
+
+    counts = df.groupby([msg_col, period]).size()
+    top_counts = counts.loc[completions].reset_index().rename(columns={0: "Size"})
+    top_counts["Completion ID"] = top_counts[msg_col].map({k: f"{i}" for i, k in enumerate(completions, start=1)})
+
+    return px.line(
+        top_counts.astype({time_col: str}),
+        x=time_col,
+        y="Size",
+        color="Completion ID",
+        hover_data=[top_counts[msg_col].str.replace("\n", "<br>")],
+        labels={time_col: f"Time, {time_interval}", "Size": f"Occurrences / {time_interval}"},
+        title=f"Completion Rates for {len(completions)} Messages",
+        **plotly_config,
+    ).update_traces(opacity=0.7)
+
+
+def plot_completion_rewards(
+    df: pd.DataFrame,
+    msg_col: str = "followup_completions",
+    reward_col: str = "followup_rewards",
+    time_col: str = "_timestamp",
+    uid_col: str = "followup_uids",
+    ntop: int = 3,
+    completions: List[str] = None,
+    completion_regex: str = None,
+) -> go.Figure:
+    """Plot completion rewards. Useful for tracking common completions and their rewards.
+
+    Args:
+        df (pd.DataFrame): Dataframe of event log.
+        msg_col (str, optional): List-like column containing completions. Defaults to 'followup_completions'.
+        reward_col (str, optional): List-like column containing rewards. Defaults to 'followup_rewards'.
+        time_col (str, optional): Column containing timestamps as pd.Datetime. Defaults to '_timestamp'.
+        ntop (int, optional): Number of completions to plot. Defaults to 20.
+        completions (List[str], optional): List of completions to plot. Defaults to None.
+        completion_regex (str, optional): Regex to match completions. Defaults to None.
+
+    """
+
+    df = (
+        df[[time_col, uid_col, msg_col, reward_col]]
+        .explode(column=[msg_col, uid_col, reward_col])
+        .rename(columns={uid_col: "UID"})
+    )
+    completion_counts = df[msg_col].value_counts()
+
+    if completions is None:
+        if completion_regex is not None:
+            completions = completion_counts[completion_counts.index.str.contains(completion_regex)].index[:ntop]
+            print(f"Using {len(completions)} completions which match {completion_regex!r}: \n{completions}")
+        else:
+            completions = completion_counts.index[:ntop]
+            print(f"Using top {len(completions)} completions: \n{completions}")
+
+    # Get ranks of completions in terms of number of occurrences
+    ranks = completion_counts.rank(method="dense", ascending=False).loc[completions].astype(int)
+
+    # Filter to only the selected completions
+    df = df.loc[df[msg_col].isin(completions)]
+    df["rank"] = df[msg_col].map(ranks).astype(str)
+    df["Total"] = df[msg_col].map(completion_counts)
+
+    return px.scatter(
+        df,
+        x=time_col,
+        y=reward_col,
+        color="rank",
+        hover_data=[msg_col, "UID", "Total"],
+        category_orders={"rank": sorted(df["rank"].unique())},
+        marginal_x="histogram",
+        marginal_y="violin",
+        labels={"rank": "Rank", reward_col: "Reward", time_col: ""},
+        title=f"Rewards for {len(completions)} Messages",
+        **plotly_config,
+        opacity=0.3,
+    )
+
+
+def plot_leaderboard(
+    df: pd.DataFrame,
+    group_on: str = "answer_uids",
+    agg_col: str = "answer_rewards",
+    agg: str = "mean",
+    ntop: int = 10,
+    alias: bool = False,
+) -> go.Figure:
+    """Plot leaderboard for a given column. By default plots the top 10 UIDs by mean reward.
+
+    Args:
+        df (pd.DataFrame): Dataframe of event log.
+        group_on (str, optional): Entities to use for grouping. Defaults to 'answer_uids'.
+        agg_col (str, optional): Column to aggregate. Defaults to 'answer_rewards'.
+        agg (str, optional): Aggregation function. Defaults to 'mean'.
+        ntop (int, optional): Number of entities to plot. Defaults to 10.
+        alias (bool, optional): Whether to use aliases for indices. Defaults to False.
+    """
+    df = df[[group_on, agg_col]].explode(column=[group_on, agg_col])
+
+    rankings = df.groupby(group_on)[agg_col].agg(agg).sort_values(ascending=False).head(ntop)
+    if alias:
+        index = rankings.index.map({name: str(i) for i, name in enumerate(rankings.index)})
+    else:
+        index = rankings.index.astype(str)
+
+    return px.bar(
+        x=rankings,
+        y=index,
+        color=rankings,
+        orientation="h",
+        labels={"x": f"{agg_col.title()}", "y": group_on, "color": ""},
+        title=f"Leaderboard for {agg_col}, top {ntop} {group_on}",
+        color_continuous_scale="BlueRed",
+        opacity=0.5,
+        hover_data=[rankings.index.astype(str)],
+        **plotly_config,
+    )
+
+
+def plot_dendrite_rates(
+    df: pd.DataFrame, uid_col: str = "answer_uids", reward_col: str = "answer_rewards", ntop: int = 20, uids: List[int] = None
+) -> go.Figure:
+    """Makes a bar chart of the success rate of dendrite calls for a given set of uids.
+
+    Args:
+        df (pd.DataFrame): Dataframe of event log.
+        uid_col (str, optional): Column containing uids. Defaults to 'answer_uids'.
+        reward_col (str, optional): Column containing rewards. Defaults to 'answer_rewards'.
+        ntop (int, optional): Number of uids to plot. Defaults to 20.
+        uids (List[int], optional): List of uids to plot. Defaults to None.
+
+    """
+
+    df = df[[uid_col, reward_col]].explode(column=[uid_col, reward_col]).rename(columns={uid_col: "UID"})
+    df["success"] = df[reward_col] != 0
+
+    if uids is None:
+        uids = df["UID"].value_counts().head(ntop).index
+    df = df.loc[df["UID"].isin(uids)]
+
+    # get total and successful dendrite calls
+    rates = df.groupby("UID").success.agg(["sum", "count"]).rename(columns={"sum": "Success", "count": "Total"})
+    rates = rates.melt(ignore_index=False).reset_index()
+    return px.bar(
+        rates.astype({"UID": str}),
+        x="value",
+        y="UID",
+        color="variable",
+        labels={"value": "Number of Calls", "variable": ""},
+        barmode="group",
+        title="Dendrite Calls by UID",
+        color_continuous_scale="Blues",
+        opacity=0.5,
+        **plotly_config,
+    )
+
+
+def plot_network_embedding(
+    df: pd.DataFrame,
+    uid_col: str = "followup_uids",
+    completion_col: str = "followup_completions",
+    ntop: int = 1,
+    uids: List[int] = None,
+) -> go.Figure:
+    """Plots a network embedding of the most common completions for a given set of uids.
+
+    Args:
+        df (pd.DataFrame): Dataframe of event log.
+
+        uid_col (str, optional): Column containing uids. Defaults to 'answer_uids'.
+        completion_col (str, optional): Column containing completions. Defaults to 'followup_completions'.
+        ntop (int, optional): Number of uids to plot. Defaults to 20.
+        hover_data (List[str], optional): Columns to include in hover data. Defaults to None.
+        uids (List[int], optional): List of uids to plot. Defaults to None.
+
+    # TODO: use value counts to use weighted similarity instead of a simple set intersection
+    """
+    top_completions = {}
+    df = df[[uid_col, completion_col]].explode(column=[uid_col, completion_col])
+
+    if uids is None:
+        uids = df[uid_col].unique()
+    # loop over UIDs and compute ntop most common completions
+    for uid in tqdm.tqdm(uids, unit="UID"):
+        c = df.loc[df[uid_col] == uid, completion_col].value_counts()
+        top_completions[uid] = set(c.index[:ntop])
+
+    a = np.zeros((len(uids), len(uids)))
+    # now compute similarity matrix as a set intersection
+    for i, uid in enumerate(uids):
+        for j, uid2 in enumerate(uids[i + 1 :], start=i + 1):
+            a[i, j] = a[j, i] = len(top_completions[uid].intersection(top_completions[uid2])) / ntop
+
+    # make a graph from the similarity matrix
+    g = nx.from_numpy_array(a)
+    z = pd.DataFrame(nx.spring_layout(g)).T.rename(columns={0: "x", 1: "y"})
+    z["UID"] = uids
+    z["top_completions"] = pd.Series(top_completions).apply(list)
+
+    # assign groups based on cliques (fully connected subgraphs)
+    cliques = {
+        uids[cc]: f"Group-{i}" if len(c) > 1 else "Other" for i, c in enumerate(nx.find_cliques(g), start=1) for cc in c
+    }
+    z["Group"] = z["UID"].map(cliques)
+
+    return px.scatter(
+        z.reset_index(),
+        x="x",
+        y="y",
+        color="Group",
+        title=f"Graph for Top {ntop} Completion Similarities",
+        color_continuous_scale="BlueRed",
+        hover_data=["UID", "top_completions"],
+        opacity=0.5,
+        **plotly_config,
+    )

requirements.txt

@@ -0,0 +1,7 @@
+wandb==0.15.3
+datasets==2.12.0
+plotly==5.14.1
+networkx==3.1
+scipy==1.10.1
+pre-commit==3.3.2
+click==8.1.3

utils.py

@@ -0,0 +1,127 @@
+# The MIT License (MIT)
+# Copyright © 2021 Yuma Rao
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import os
+import tqdm
+import wandb
+import pandas as pd
+from pandas.api.types import is_list_like
+
+from typing import List, Dict, Any, Union
+
+
+def get_runs(project: str = "openvalidators", filters: Dict[str, Any] = None, return_paths: bool = False) -> List:
+    """Download runs from wandb.
+
+    Args:
+        project (str): Name of the project. Defaults to 'openvalidators' (community project)
+        filters (Dict[str, Any], optional): Optional run filters for wandb api. Defaults to None.
+        return_paths (bool, optional): Return only run paths. Defaults to False.
+
+    Returns:
+        List[wandb.apis.public.Run]: List of runs or run paths (List[str]).
+    """
+    api = wandb.Api()
+    wandb.login()
+
+    runs = api.runs(project, filters=filters)
+    if return_paths:
+        return [os.path.join(run.entity, run.project, run.id) for run in runs]
+    else:
+        return runs
+
+
+def download_data(run_path: Union[str, List] = None, timeout: float = 600) -> pd.DataFrame:
+    """Download data from wandb.
+
+    Args:
+        run_path (Union[str, List], optional): Path to run or list of paths. Defaults to None.
+        timeout (float, optional): Timeout for wandb api. Defaults to 600.
+
+    Returns:
+        pd.DataFrame: Dataframe of event log.
+    """
+    api = wandb.Api(timeout=timeout)
+    wandb.login()
+
+    if isinstance(run_path, str):
+        run_path = [run_path]
+
+    frames = []
+    total_events = 0
+    pbar = tqdm.tqdm(sorted(run_path), desc="Loading history from wandb", total=len(run_path), unit="run")
+    for path in pbar:
+        run = api.run(path)
+
+        frame = pd.DataFrame(list(run.scan_history()))
+        frames.append(frame)
+        total_events += len(frame)
+
+        pbar.set_postfix({"total_events": total_events})
+
+    df = pd.concat(frames)
+    # Convert timestamp to datetime.
+    df._timestamp = pd.to_datetime(df._timestamp, unit="s")
+    df.sort_values("_timestamp", inplace=True)
+
+    return df
+
+
+def load_data(path: str, nrows: int = None):
+    """Load data from csv."""
+    df = pd.read_csv(path, nrows=nrows)
+    # filter out events with missing step length
+    df = df.loc[df.step_length.notna()]
+
+    # detect list columns which as stored as strings
+    list_cols = [c for c in df.columns if df[c].dtype == "object" and df[c].str.startswith("[").all()]
+    # convert string representation of list to list
+    df[list_cols] = df[list_cols].applymap(eval, na_action='ignore')
+
+    return df
+
+
+def explode_data(df: pd.DataFrame, list_cols: List[str] = None, list_len: int = None) -> pd.DataFrame:
+    """Explode list columns in dataframe so that each element in the list is a separate row.
+
+    Args:
+        df (pd.DataFrame): Dataframe of event log.
+        list_cols (List[str], optional): List of columns to explode. Defaults to None.
+        list_len (int, optional): Length of list. Defaults to None.
+
+    Returns:
+        pd.DataFrame: Dataframe with exploded list columns.
+    """
+    if list_cols is None:
+        list_cols = [c for c in df.columns if df[c].apply(is_list_like).all()]
+        print(f"Exploding {len(list_cols)}) list columns with {list_len} elements: {list_cols}")
+    if list_len:
+        list_cols = [c for c in list_cols if df[c].apply(len).unique()[0] == list_len]
+        print(f"Exploding {len(list_cols)}) list columns with {list_len} elements: {list_cols}")
+
+    return df.explode(column=list_cols)
+
+
+def get_list_col_lengths(df: pd.DataFrame) -> Dict[str, int]:
+    """Helper function to get the length of list columns."""
+    list_col_lengths = {c: sorted(df[c].apply(len).unique()) for c in df.columns if df[c].apply(is_list_like).all()}
+    varying_lengths = {c: v for c, v in list_col_lengths.items() if len(v) > 1}
+
+    if len(varying_lengths) > 0:
+        print(f"The following columns have varying lengths: {varying_lengths}")
+
+    return {c: v[0] for c, v in list_col_lengths.items()}