File size: 4,516 Bytes
f740333 b0a6f33 6db6faf b0a6f33 a66535a 6db6faf b0a6f33 f740333 b0a6f33 f740333 a66535a f740333 a66535a f740333 6db6faf f740333 a66535a f740333 a66535a f740333 a66535a 50faa2d 6db6faf 19c9463 6db6faf |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 |
import pandas as pd
import gradio as gr
import matplotlib.pyplot as plt
import seaborn as sns
from typing import Tuple
import plotly.express as px
VOLUME_FACTOR_REGULARIZATION = 0.5
UNSCALED_WEIGHTED_ACCURACY_INTERVAL = (-0.5, 100.5)
SCALED_WEIGHTED_ACCURACY_INTERVAL = (0, 1)
# tools palette as dictionary
tools_palette = {
"prediction-request-reasoning": "darkorchid",
"claude-prediction-offline": "rebeccapurple",
"prediction-request-reasoning-claude": "slateblue",
"prediction-request-rag-claude": "steelblue",
"prediction-online": "darkcyan",
"prediction-offline": "mediumaquamarine",
"claude-prediction-online": "mediumseagreen",
"prediction-online-sme": "yellowgreen",
"prediction-url-cot-claude": "gold",
"prediction-offline-sme": "orange",
"prediction-request-rag": "chocolate",
}
HEIGHT = 400
WIDTH = 1100
def scale_value(
value: float,
min_max_bounds: Tuple[float, float],
scale_bounds: Tuple[float, float] = (0, 1),
) -> float:
"""Perform min-max scaling on a value."""
min_, max_ = min_max_bounds
current_range = max_ - min_
# normalize between 0-1
std = (value - min_) / current_range
# scale between min_bound and max_bound
min_bound, max_bound = scale_bounds
target_range = max_bound - min_bound
return std * target_range + min_bound
def get_weighted_accuracy(row, global_requests: int):
"""Function to compute the weighted accuracy of a tool"""
return scale_value(
(
row["tool_accuracy"]
+ (row["total_requests"] / global_requests) * VOLUME_FACTOR_REGULARIZATION
),
UNSCALED_WEIGHTED_ACCURACY_INTERVAL,
SCALED_WEIGHTED_ACCURACY_INTERVAL,
)
def compute_weighted_accuracy(tools_accuracy: pd.DataFrame):
global_requests = tools_accuracy.total_requests.sum()
tools_accuracy["weighted_accuracy"] = tools_accuracy.apply(
lambda x: get_weighted_accuracy(x, global_requests), axis=1
)
return tools_accuracy
def plot_tools_accuracy_graph(tools_accuracy_info: pd.DataFrame):
tools_accuracy_info = tools_accuracy_info.sort_values(
by="tool_accuracy", ascending=False
)
plt.figure(figsize=(25, 10))
plot = sns.barplot(
tools_accuracy_info,
x="tool_accuracy",
y="tool",
hue="tool",
dodge=False,
palette=tools_palette,
)
plt.xlabel("Mech tool_accuracy (%)", fontsize=20)
plt.ylabel("tool", fontsize=20)
plt.tick_params(axis="y", labelsize=12)
return gr.Plot(value=plot.get_figure())
def plot_tools_accuracy_rotated_graph(tools_accuracy_info: pd.DataFrame):
tools_accuracy_info = tools_accuracy_info.sort_values(
by="tool_accuracy", ascending=False
)
fig = px.bar(
tools_accuracy_info,
x="tool",
y="tool_accuracy",
color="tool",
color_discrete_map=tools_palette,
)
fig.update_layout(
xaxis_title="Tool",
yaxis_title="Mech tool_accuracy (%)",
)
fig.update_layout(width=WIDTH, height=HEIGHT)
# fig.update_xaxes(tickangle=45)
fig.update_xaxes(showticklabels=False)
return gr.Plot(
value=fig,
)
def plot_tools_weighted_accuracy_graph(tools_accuracy_info: pd.DataFrame):
tools_accuracy_info = tools_accuracy_info.sort_values(
by="weighted_accuracy", ascending=False
)
# Create the Seaborn bar plot
# sns.set_theme(palette="viridis")
plt.figure(figsize=(25, 10))
plot = sns.barplot(
tools_accuracy_info,
x="weighted_accuracy",
y="tool",
hue="tool",
dodge=False,
palette=tools_palette,
)
plt.xlabel("Weighted accuracy metric", fontsize=20)
plt.ylabel("tool", fontsize=20)
plt.tick_params(axis="y", labelsize=12)
return gr.Plot(value=plot.get_figure())
def plot_tools_weighted_accuracy_rotated_graph(
tools_accuracy_info: pd.DataFrame,
) -> gr.Plot:
tools_accuracy_info = tools_accuracy_info.sort_values(
by="weighted_accuracy", ascending=False
)
fig = px.bar(
tools_accuracy_info,
x="tool",
y="weighted_accuracy",
color="tool",
color_discrete_map=tools_palette,
)
fig.update_layout(
xaxis_title="Tool",
yaxis_title="Weighted accuracy metric",
)
fig.update_layout(width=WIDTH, height=HEIGHT)
# fig.update_xaxes(tickangle=45)
fig.update_xaxes(showticklabels=False)
return gr.Plot(
value=fig,
)
|