Spaces:
Sleeping
Sleeping
Boubou78000
commited on
Commit
·
0a88a11
1
Parent(s):
2c3f7c0
app.py
CHANGED
@@ -5,6 +5,7 @@ from datasets import load_dataset
|
|
5 |
from transformers import AutoTokenizer
|
6 |
|
7 |
def ReturnTokens(dataset_name, tokenizer_name="openai-community/gpt2", split="train"):
|
|
|
8 |
# Initialize tokenizer
|
9 |
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
|
10 |
|
@@ -34,26 +35,21 @@ with gr.Blocks(title="Dataset token counter") as app:
|
|
34 |
tokenizer = gr.Textbox(label="Tokenizer", elem_id="tokenizer", info="", placeholder="openai-community/gpt2", value="openai-community/gpt2")
|
35 |
split = gr.Textbox(label="Split (default: train)", elem_id="split", info="", placeholder="train", value="train")
|
36 |
tokens = gr.Label(label="Tokens", elem_id="tokens")
|
37 |
-
prompt.submit().success(
|
38 |
-
ReturnTokens,
|
39 |
-
inputs=[prompt,tokenizer,split],
|
40 |
-
outputs=[tokens]
|
41 |
-
)
|
42 |
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
|
59 |
app.launch()
|
|
|
5 |
from transformers import AutoTokenizer
|
6 |
|
7 |
def ReturnTokens(dataset_name, tokenizer_name="openai-community/gpt2", split="train"):
|
8 |
+
|
9 |
# Initialize tokenizer
|
10 |
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
|
11 |
|
|
|
35 |
tokenizer = gr.Textbox(label="Tokenizer", elem_id="tokenizer", info="", placeholder="openai-community/gpt2", value="openai-community/gpt2")
|
36 |
split = gr.Textbox(label="Split (default: train)", elem_id="split", info="", placeholder="train", value="train")
|
37 |
tokens = gr.Label(label="Tokens", elem_id="tokens")
|
|
|
|
|
|
|
|
|
|
|
38 |
|
39 |
+
gr.on(
|
40 |
+
triggers=[
|
41 |
+
prompt.submit,
|
42 |
+
tokenizer.submit,
|
43 |
+
split.submit,
|
44 |
+
],
|
45 |
+
fn=ReturnTokens,
|
46 |
+
inputs=[
|
47 |
+
prompt,
|
48 |
+
tokenizer,
|
49 |
+
split
|
50 |
+
],
|
51 |
+
outputs=[tokens],
|
52 |
+
api_name="run",
|
53 |
+
)
|
54 |
|
55 |
app.launch()
|