Spaces:

bigscience-data
/

bigscience-tokenizer

Runtime error

christopher commited on May 5, 2022

Commit

8cc0de9

1 Parent(s): f003591

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -2,7 +2,7 @@ import streamlit as st
 from tokenizers.tools import EncodingVisualizer
 from transformers import AutoTokenizer
 st.set_page_config(page_title="BigScience Tokenizer", page_icon='👩‍💻', layout="wide")
-tokenizer = AutoTokenizer.from_pretrained('bigscience/tokenizer', use_auth_token=True)
 visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False)
 text = st.text_area(label="", placeholder="Text to tokenize")
 button_clicked = st.button("Tokenize")

 from tokenizers.tools import EncodingVisualizer
 from transformers import AutoTokenizer
 st.set_page_config(page_title="BigScience Tokenizer", page_icon='👩‍💻', layout="wide")
+tokenizer = AutoTokenizer.from_pretrained('bigscience/tokenizer', use_auth_token=st.secrets["AUTH_TOKEN"])
 visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False)
 text = st.text_area(label="", placeholder="Text to tokenize")
 button_clicked = st.button("Tokenize")