Spaces:
Sleeping
Sleeping
File size: 1,150 Bytes
98985f3 b5dfd52 98985f3 70fa933 98985f3 b5dfd52 70fa933 60a0592 70fa933 98985f3 9937e65 b5dfd52 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 |
from transformers import AutoTokenizer
import gradio as gr
gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
yi_tokenizer = AutoTokenizer.from_pretrained("01-ai/Yi-34B")
yayi2_tokenizer = AutoTokenizer.from_pretrained("wenge-research/yayi2-30b")
phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
def tokenize(input_text):
gpt2_tokens = gpt2_tokenizer(input_text)["input_ids"]
gpt_neox_tokens = gpt_neox_tokenizer(input_text)["input_ids"]
llama_tokens = llama_tokenizer(input_text)["input_ids"]
yi_tokens = llama_tokenizer(input_text)["input_ids"]
yayi2_tokens = yayi2_tokenizer(input_text)["input_ids"]
phi2_tokens = phi2_tokenizer(input_text)["input_ids"]
return f"GPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nLLaMa: {len(llama_tokens)}\nYi: {len(yi_tokens)}\nYayi-2: {len(yayi2_tokens)}\nPhi-2: {len(phi2_tokens)}"
iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=7), outputs="text")
iface.launch() |