plotting.py

import plotly.express as px
import plotly.graph_objects as go

from utils import DEEPLITE_LIGHT_BLUE_HEX, load_yolobench_data


df, pareto_indices = load_yolobench_data()

METRIC_NAME_MAPPING = {
    '[email protected]': 'mAP_0.5',
    '[email protected]:0.95': 'mAP_0.5:0.95',
    'Precision': 'precision',
    'Recall': 'recall',
}

METRIC_KEYS_TO_NAMES = {v: k for k, v in METRIC_NAME_MAPPING.items()}


LATENCY_KEYS = {
    'Raspberry Pi 4 Model B (CPU, TFLite, FP32)': 'raspi4_tflite_latency',
    'Raspberry Pi 4 Model B (CPU, ONNX Runtime, FP32)': 'pi4_ort_latency',
    'Raspberry Pi 5 Model B (CPU, ONNX Runtime, FP32)': 'pi5_ort_latency',
    'Jetson Nano (GPU, ONNX Runtime, FP32)': 'nano_gpu_latency',
    'Intel® Core™i7-10875H (CPU, OpenVINO, FP32)': 'openvino_latency',
    'Khadas VIM3 (NPU, INT16)': 'vim3_latency',
    'Orange Pi 5 (NPU, FP16)': 'orange_pi_latency',
    'NVIDIA A40 (GPU, TensorRT, FP32)': 'a40_trt_latency',
}

LATENCY_KEYS_TO_NAMES = {v: k for k, v in LATENCY_KEYS.items()}

DATASET_TAGS = {
    'PASCAL VOC': 'voc',
    'SKU-110K': 'sku',
    'WIDERFACE': 'wider',
    'COCO': 'coco',
}

DATASET_TAGS_TO_NAMES = {v: k for k, v in DATASET_TAGS.items()}


def get_scatter_plot(
    dataset_tag,
    metric_tag,
    latency_key,
    model_family_coloring=True,
    add_pareto_frontier=False,
    plot_pareto_only=False,
    log_axis=False,
):
    fig_opts, layout_opts = {
        'opacity': 0.5,
        'color_discrete_sequence': [DEEPLITE_LIGHT_BLUE_HEX],
    }, {}
    if model_family_coloring:
        fig_opts = {
            'color': 'model_family',
            'opacity': 0.75,
            'color_discrete_sequence': px.colors.qualitative.Plotly,
        }
        layout_opts = {
            'legend': dict(
                title='Model family<br>(click to toggle)',
            )
        }

    frontier = None
    if plot_pareto_only:
        metric_key = f'{metric_tag}_{dataset_tag}'
        frontier = pareto_indices[metric_key][latency_key]

    fig = px.scatter(
        df if frontier is None else df.iloc[frontier, :],
        x=latency_key,
        y=f'{metric_tag}_{dataset_tag}',
        title=f'{METRIC_KEYS_TO_NAMES[metric_tag]}-latency scatter plot',
        hover_data={
            'model_name': True,
            'model_family': False,
            latency_key: ':.2f',
            f'{metric_tag}_{dataset_tag}': ':.2f',
        },
        labels={
            'model_name': 'Model name',
            latency_key: 'Latency',
            f'{metric_tag}_{dataset_tag}': METRIC_KEYS_TO_NAMES[metric_tag],
        },
        template='plotly_white',
        **fig_opts,
    )
    if log_axis:
        fig.update_xaxes(type='log')

    fig.update_layout(
        height=600,
        modebar_remove=[
            'lasso',
            'autoscale',
            'zoomin',
            'zoomout',
            'select2d',
            'select',
        ],
        xaxis_title=f'{LATENCY_KEYS_TO_NAMES[latency_key]} latency, ms',
        yaxis_title=f"{METRIC_KEYS_TO_NAMES[metric_tag]}",
        xaxis=dict(
            rangeslider=dict(
                visible=True,
                bgcolor=DEEPLITE_LIGHT_BLUE_HEX,
                thickness=0.02,
            ),
        ),
        yaxis=dict(
            fixedrange=False,
        ),
        hoverlabel=dict(
            # bgcolor="white",
            font_size=14,
            font_family='Source Sans Pro',
        ),
        **layout_opts,
    )
    if add_pareto_frontier:
        fig = pareto_frontier_layer(fig, dataset_tag, metric_tag, latency_key)
    fig.update_layout(autosize=True)
    return fig


def get_comparison_plot(
    dataset_tag,
    metric_tag,
    latency_keys,
    log_axis=False,
    remove_marker=False,
):
    if len(latency_keys) == 0:
        layout_opts = {
            "annotations": [
                {
                    "text": "Please select at least 2 hardware targets to compare!",
                    "showarrow": False,
                    "font": {"size": 28},
                }
            ]
        }
    else:
        layout_opts = {
            'legend': dict(
                title='Hardware targets selected<br>(click to toggle)',
            )
        }

    fig = go.Figure(
        layout=go.Layout(
            title=go.layout.Title(
                text=f'{METRIC_KEYS_TO_NAMES[metric_tag]}-latency Pareto Optimal Models'
            )
        )
    )

    if remove_marker:
        mode = 'lines'
    else:
        mode = 'lines+markers'

    for latency_key in latency_keys:
        metric_key = f'{metric_tag}_{dataset_tag}'
        frontier = pareto_indices[metric_key][latency_key]
        df_pareto = df.iloc[frontier, :]
        model_name = df_pareto['model_name']
        fig.add_trace(
            go.Scatter(
                x=df_pareto[latency_key],
                y=df_pareto[f'{metric_tag}_{dataset_tag}'],
                name=LATENCY_KEYS_TO_NAMES[latency_key],
                mode=mode,
                customdata=model_name,
                hovertemplate='<b>model name:%{customdata}</b><br>latency:%{x:.2f} <br>metric: %{y:.2f} ',
            )
        )

    if log_axis:
        fig.update_xaxes(type='log')
        x_axis_title = f'Inference latency, ms (BS=1, log scale)'
    else:
        x_axis_title = f'Inference latency, ms (BS=1)'

    fig.update_layout(
        height=600,
        plot_bgcolor='white',
        modebar_remove=[
            'lasso',
            'autoscale',
            'zoomin',
            'zoomout',
            'select2d',
            'select',
        ],
        xaxis_title=x_axis_title,
        yaxis_title=f"{METRIC_KEYS_TO_NAMES[metric_tag]}",
        xaxis=dict(
            rangeslider=dict(
                visible=True,
                bgcolor=DEEPLITE_LIGHT_BLUE_HEX,
                thickness=0.02,
            ),
        ),
        yaxis=dict(
            fixedrange=False,
        ),
        hoverlabel=dict(
            # bgcolor="white",
            font_size=14,
            font_family='Source Sans Pro',
        ),
        **layout_opts,
    )
    fig.update_layout(autosize=True, template="plotly_white")

    return fig


def pareto_frontier_layer(
    fig,
    dataset_tag,
    metric_tag,
    latency_key,
):
    metric_key = f'{metric_tag}_{dataset_tag}'
    frontier = pareto_indices[metric_key][latency_key]
    fig.add_trace(
        go.Scatter(
            x=df.iloc[frontier, :][latency_key],
            y=df.iloc[frontier, :][metric_key],
            mode='lines+markers',
            opacity=0.5,
            line=go.scatter.Line(color='grey'),
            showlegend=False,
            name=metric_key,
            marker=dict(symbol=['circle']),
        )
    )
    return fig


def create_yolobench_plots(
    dataset_name,
    hardware_name,
    metric_name,
    vis_options,
    table_mode,
):
    model_family_coloring = 'Model family' in vis_options
    add_pareto_frontier = 'Highlight Pareto' in vis_options
    plot_pareto_only = 'Show Pareto only' in vis_options
    log_axis = 'Log x-axis' in vis_options
    fig = get_scatter_plot(
        DATASET_TAGS[dataset_name],
        METRIC_NAME_MAPPING[metric_name],
        LATENCY_KEYS[hardware_name],
        model_family_coloring,
        add_pareto_frontier,
        plot_pareto_only,
        log_axis,
    )
    pareto_table = get_pareto_table(
        dataset_name, hardware_name, metric_name, expand_table='Show all' in table_mode
    )
    return fig, pareto_table


def pareto_frontier_layer(
    fig,
    dataset_tag,
    metric_tag,
    latency_key,
):
    metric_key = f'{metric_tag}_{dataset_tag}'
    frontier = pareto_indices[metric_key][latency_key]
    fig.add_trace(
        go.Scatter(
            x=df.iloc[frontier, :][latency_key],
            y=df.iloc[frontier, :][metric_key],
            mode='lines',
            opacity=0.5,
            line=go.scatter.Line(color='grey'),
            showlegend=False,
            name=metric_key,
        )
    )
    return fig


def create_comparison_plot(dataset_name, hardware_list, metric_name, vis_options):
    log_axis = 'Log x-axis' in vis_options
    remove_marker = 'Remove datapoint markers' in vis_options

    latency_keys = []
    for hardware_name in hardware_list:
        latency_keys.append(LATENCY_KEYS[hardware_name])

    fig = get_comparison_plot(
        DATASET_TAGS[dataset_name],
        METRIC_NAME_MAPPING[metric_name],
        latency_keys,
        log_axis,
        remove_marker,
    )
    return fig


def get_pareto_table(
    dataset_name,
    hardware_name,
    metric_name,
    expand_table=False,
):
    dataset_tag = DATASET_TAGS[dataset_name]
    metric_tag = METRIC_NAME_MAPPING[metric_name]
    latency_key = LATENCY_KEYS[hardware_name]
    metric_key = f'{metric_tag}_{dataset_tag}'

    latency_key_final = f'{LATENCY_KEYS_TO_NAMES[latency_key]} latency, ms'
    metric_key_final = METRIC_KEYS_TO_NAMES[metric_tag]

    frontier = pareto_indices[metric_key][latency_key]
    table_df = df.iloc[frontier, :][['model_name', metric_key, latency_key]]
    table_df['Input resolution (px)'] = table_df['model_name'].apply(
        lambda name: name.split('_')[-1]
    )
    table_df['Model name'] = table_df['model_name'].apply(
        lambda name: name.split('_')[0]
    )
    table_df[metric_key_final] = table_df[metric_key].apply(lambda val: round(val, 3))
    table_df[latency_key_final] = table_df[latency_key].apply(lambda val: round(val, 2))

    def make_clickable(url, name):
        return f'<a href="{url}">{name}</a>'

    if dataset_name == 'COCO':
        table_df['Download link'] = table_df['model_name'].apply(
            lambda name: f'https://download.deeplite.ai/zoo/models/YOLOBench/{name.split("_")[0]}_640.pt'
        )
        table_df['Download link'] = table_df.apply(
            lambda x: make_clickable(x['Download link'], 'Weights download'), axis=1
        )
    else:
        table_df['Download link'] = table_df['model_name'].apply(
            lambda s: 'Coming soon'
        )

    table_df = table_df[
        [
            'Model name',
            'Input resolution (px)',
            metric_key_final,
            latency_key_final,
            'Download link',
        ]
    ].sort_values(by=metric_key_final, ascending=False)
    if not expand_table:
        table_df = table_df.iloc[:10, :]

    table_df = table_df.to_html(
        classes='table', escape=False, render_links=True, index=False
    )

    return table_df