Spaces:
Runtime error
Runtime error
Commit
Β·
f003591
1
Parent(s):
d70aac7
Update app.py
Browse files
app.py
CHANGED
@@ -2,4 +2,9 @@ import streamlit as st
|
|
2 |
from tokenizers.tools import EncodingVisualizer
|
3 |
from transformers import AutoTokenizer
|
4 |
st.set_page_config(page_title="BigScience Tokenizer", page_icon='π©βπ»', layout="wide")
|
|
|
|
|
5 |
text = st.text_area(label="", placeholder="Text to tokenize")
|
|
|
|
|
|
|
|
2 |
from tokenizers.tools import EncodingVisualizer
|
3 |
from transformers import AutoTokenizer
|
4 |
st.set_page_config(page_title="BigScience Tokenizer", page_icon='π©βπ»', layout="wide")
|
5 |
+
tokenizer = AutoTokenizer.from_pretrained('bigscience/tokenizer', use_auth_token=True)
|
6 |
+
visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False)
|
7 |
text = st.text_area(label="", placeholder="Text to tokenize")
|
8 |
+
button_clicked = st.button("Tokenize")
|
9 |
+
if text or button_clicked:
|
10 |
+
st.write(visualizer(text), unsafe_allow_html=True)
|