src/validator_states.py

import statistics
from datetime import timedelta

import gradio as gr
import pandas as pd
from packaging import version

from wandb_data import get_current_runs, Run
from chain_data import VTRUST, UPDATED

AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD = 180  # 3 minutes
ETA_WARNING_THRESHOLD = 43200  # 12 hours
UPDATED_WARNING_THRESHOLD = 1000
VTRUST_WARNING_THRESHOLD = 0.75


def get_latest_version(runs: list[Run]) -> str:
    latest_version = version.parse("0.0.0")
    for run in runs:
        current_version = version.parse(run.version)
        if current_version > latest_version:
            latest_version = current_version
    return str(latest_version)


def colorize(val, color: str) -> str:
    return f"<span style='color: {color}'>{val}</span>"


def create_validator_states() -> gr.Dataframe:
    data: list[list] = []
    runs = sorted(get_current_runs(), key=lambda run: run.uid)
    winners = [run.winner_uid for run in runs if run.winner_uid]
    winner_uid_mode = statistics.mode(winners) if winners else None
    latest_version = get_latest_version(runs)
    for run in runs:
        vtrust = VTRUST.get(run.uid, 0)
        updated = UPDATED.get(run.uid, 0)
        data.append([
            run.uid,
            run.name,
            colorize(run.version, "springgreen" if run.version == latest_version else "red"),
            colorize(run.status.name(), run.status.color()),
            colorize(run.winner_uid, "springgreen" if winner_uid_mode and run.winner_uid == winner_uid_mode else "orange" if run.winner_uid else "gray"),
            f"{min(run.total_submissions, len(run.submissions) + len(run.invalid_submissions))}/{run.total_submissions}",
            len(run.invalid_submissions),
            colorize(f"{timedelta(seconds=int(run.average_benchmark_time))}", "orange" if run.average_benchmark_time > AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD else "springgreen" if run.average_benchmark_time > 0 else "gray"),
            colorize(f"{timedelta(seconds=run.eta)}", "orange" if run.eta > ETA_WARNING_THRESHOLD else "springgreen" if run.eta > 0 else "gray"),
            colorize(f"{vtrust:.4f}", "springgreen" if vtrust > VTRUST_WARNING_THRESHOLD else "red"),
            colorize(updated, "springgreen" if updated < UPDATED_WARNING_THRESHOLD else "red"),
        ])

    return gr.Dataframe(
        pd.DataFrame(data, columns=["UID", "Name", "Version", "Status", "Winner", "Tested", "Invalid", "Avg. Benchmark Time", "ETA", "VTrust", "Updated"]),
        datatype=["number", "markdown", "markdown", "markdown", "markdown", "number", "number", "markdown", "markdown", "markdown", "markdown"],
        interactive=False,
    )