Spaces:
Runtime error
Runtime error
Add error handling for no tokenizer selected
Browse files
app.py
CHANGED
@@ -84,30 +84,34 @@ st.table(st.session_state.examplesdf)
|
|
84 |
# Create a distribution plot for token density across selected tokenizers
|
85 |
import plotly.figure_factory as ff
|
86 |
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
# Creating the distplot with optional histogram
|
91 |
-
fig = ff.create_distplot(
|
92 |
-
hist_data, selected_tokenizers, show_hist=False, show_rug=False
|
93 |
-
)
|
94 |
-
fig.update_layout(
|
95 |
-
title="Token Distribution Density",
|
96 |
-
xaxis_title="Number of Tokens",
|
97 |
-
yaxis_title="Density",
|
98 |
-
height=500,
|
99 |
-
)
|
100 |
-
st.plotly_chart(fig, use_container_width=True)
|
101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
102 |
|
103 |
-
tokenizer_to_num_tokens = {
|
104 |
-
|
105 |
-
}
|
106 |
|
107 |
-
fig = go.Figure()
|
108 |
-
for tokenizer_name in selected_tokenizers:
|
109 |
-
|
110 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
111 |
)
|
112 |
-
fig.update_layout(title="Token Count Variability")
|
113 |
-
st.plotly_chart(fig)
|
|
|
84 |
# Create a distribution plot for token density across selected tokenizers
|
85 |
import plotly.figure_factory as ff
|
86 |
|
87 |
+
if selected_tokenizers:
|
88 |
+
# Collecting data for all selected tokenizers
|
89 |
+
hist_data = [val_data[tokenizer].dropna() for tokenizer in selected_tokenizers]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
90 |
|
91 |
+
# Creating the distplot with optional histogram
|
92 |
+
fig = ff.create_distplot(
|
93 |
+
hist_data, selected_tokenizers, show_hist=False, show_rug=False
|
94 |
+
)
|
95 |
+
fig.update_layout(
|
96 |
+
title="Token Distribution Density",
|
97 |
+
xaxis_title="Number of Tokens",
|
98 |
+
yaxis_title="Density",
|
99 |
+
height=500,
|
100 |
+
)
|
101 |
+
st.plotly_chart(fig, use_container_width=True)
|
102 |
|
103 |
+
tokenizer_to_num_tokens = {
|
104 |
+
name: val_data[name].tolist() for name in selected_tokenizers
|
105 |
+
}
|
106 |
|
107 |
+
fig = go.Figure()
|
108 |
+
for tokenizer_name in selected_tokenizers:
|
109 |
+
fig.add_trace(
|
110 |
+
go.Box(y=tokenizer_to_num_tokens[tokenizer_name], name=tokenizer_name)
|
111 |
+
)
|
112 |
+
fig.update_layout(title="Token Count Variability")
|
113 |
+
st.plotly_chart(fig)
|
114 |
+
else:
|
115 |
+
st.error(
|
116 |
+
"No tokenizers selected. Please select at least one tokenizer to view the distribution plot."
|
117 |
)
|
|
|
|