christopher commited on
Commit
f003591
Β·
1 Parent(s): d70aac7

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +5 -0
app.py CHANGED
@@ -2,4 +2,9 @@ import streamlit as st
2
  from tokenizers.tools import EncodingVisualizer
3
  from transformers import AutoTokenizer
4
  st.set_page_config(page_title="BigScience Tokenizer", page_icon='πŸ‘©β€πŸ’»', layout="wide")
 
 
5
  text = st.text_area(label="", placeholder="Text to tokenize")
 
 
 
 
2
  from tokenizers.tools import EncodingVisualizer
3
  from transformers import AutoTokenizer
4
  st.set_page_config(page_title="BigScience Tokenizer", page_icon='πŸ‘©β€πŸ’»', layout="wide")
5
+ tokenizer = AutoTokenizer.from_pretrained('bigscience/tokenizer', use_auth_token=True)
6
+ visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False)
7
  text = st.text_area(label="", placeholder="Text to tokenize")
8
+ button_clicked = st.button("Tokenize")
9
+ if text or button_clicked:
10
+ st.write(visualizer(text), unsafe_allow_html=True)