5w4n commited on
Commit
9acdffa
1 Parent(s): 6841f1c

Add error handling for no tokenizer selected

Browse files
Files changed (1) hide show
  1. app.py +27 -23
app.py CHANGED
@@ -84,30 +84,34 @@ st.table(st.session_state.examplesdf)
84
  # Create a distribution plot for token density across selected tokenizers
85
  import plotly.figure_factory as ff
86
 
87
- # Collecting data for all selected tokenizers
88
- hist_data = [val_data[tokenizer].dropna() for tokenizer in selected_tokenizers]
89
-
90
- # Creating the distplot with optional histogram
91
- fig = ff.create_distplot(
92
- hist_data, selected_tokenizers, show_hist=False, show_rug=False
93
- )
94
- fig.update_layout(
95
- title="Token Distribution Density",
96
- xaxis_title="Number of Tokens",
97
- yaxis_title="Density",
98
- height=500,
99
- )
100
- st.plotly_chart(fig, use_container_width=True)
101
 
 
 
 
 
 
 
 
 
 
 
 
102
 
103
- tokenizer_to_num_tokens = {
104
- name: val_data[name].tolist() for name in selected_tokenizers
105
- }
106
 
107
- fig = go.Figure()
108
- for tokenizer_name in selected_tokenizers:
109
- fig.add_trace(
110
- go.Box(y=tokenizer_to_num_tokens[tokenizer_name], name=tokenizer_name)
 
 
 
 
 
 
111
  )
112
- fig.update_layout(title="Token Count Variability")
113
- st.plotly_chart(fig)
 
84
  # Create a distribution plot for token density across selected tokenizers
85
  import plotly.figure_factory as ff
86
 
87
+ if selected_tokenizers:
88
+ # Collecting data for all selected tokenizers
89
+ hist_data = [val_data[tokenizer].dropna() for tokenizer in selected_tokenizers]
 
 
 
 
 
 
 
 
 
 
 
90
 
91
+ # Creating the distplot with optional histogram
92
+ fig = ff.create_distplot(
93
+ hist_data, selected_tokenizers, show_hist=False, show_rug=False
94
+ )
95
+ fig.update_layout(
96
+ title="Token Distribution Density",
97
+ xaxis_title="Number of Tokens",
98
+ yaxis_title="Density",
99
+ height=500,
100
+ )
101
+ st.plotly_chart(fig, use_container_width=True)
102
 
103
+ tokenizer_to_num_tokens = {
104
+ name: val_data[name].tolist() for name in selected_tokenizers
105
+ }
106
 
107
+ fig = go.Figure()
108
+ for tokenizer_name in selected_tokenizers:
109
+ fig.add_trace(
110
+ go.Box(y=tokenizer_to_num_tokens[tokenizer_name], name=tokenizer_name)
111
+ )
112
+ fig.update_layout(title="Token Count Variability")
113
+ st.plotly_chart(fig)
114
+ else:
115
+ st.error(
116
+ "No tokenizers selected. Please select at least one tokenizer to view the distribution plot."
117
  )