app.py

import os

import gradio as gr
import tiktoken


os.environ["TIKTOKEN_CACHE_DIR"] = ""

encoding = tiktoken.get_encoding("cl100k_base")

enc_mapping = {
    "gpt-4": "cl100k_base", "gpt-3.5-turbo(chatgpt)": "cl100k_base", "text-embedding-ada-002": "cl100k_base", "Codex": "p50k_base", "text-davinci-002": "p50k_base", "text-davinci-003": "p50k_base", "gpt3": "r50k_base", "gpt2": "r50k_base"
}


def tokenize(text, model):
    encoding = tiktoken.get_encoding(enc_mapping[model])
    enc = encoding.encode(text)
    return len(enc), enc


title = "GPT Token"
description = "This demo uses <a href='https://github.com/openai/tiktoken' target='_blank'>tiktoken</a> to calculate the token number needed for GPT models."

iface = gr.Interface(fn=tokenize,
                     inputs=[
                         gr.Textbox(label="input sequence"),
                         gr.Radio(choices=["gpt-4", "gpt-3.5-turbo(chatgpt)", "text-embedding-ada-002", "Codex", "text-davinci-002", "text-davinci-003", "gpt3", "gpt2"], value="gpt-3.5-turbo(chatgpt)", label="model")],
                     outputs=[gr.Textbox(label="token number"), gr.Textbox(
                         label="token sequence")],
                     title=title,
                     description=description,
                     allow_flagging='never')
iface.launch(share=False, debug=True)