Add application file

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Athens NLP Summer School
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,12 @@
+---
+title: Tokenization Playground
+emoji: 📝
+colorFrom: indigo
+colorTo: purple
+sdk: gradio
+pinned: false
+short_description: Compare different tokenizers
+---
+
+# tokenization_playground
+Link to source code: https://github.com/athnlp/tokenization_playground

app.py

@@ -0,0 +1,24 @@
+import os
+
+import gradio as gr
+from huggingface_hub import login
+
+from playground_app import demo as playground_tab
+
+auth_token = os.environ.get("HF_TOKEN", None)
+if auth_token:
+    login(token=auth_token)
+
+
+title = """
+<div align="center">
+    <span>Tokenization Playground</span>
+</div>
+"""
+
+with gr.Blocks() as demo:
+    gr.HTML(f"<h1 style='text-align: center; margin-bottom: 1rem'>{title}</h1>")
+    playground_tab.render()
+
+if __name__ == "__main__":
+    demo.launch(share=True)

playground_app.py

@@ -0,0 +1,190 @@
+import json
+from functools import lru_cache
+from typing import Any
+
+import gradio as gr
+import pandas as pd
+from loguru import logger
+
+from playground_examples import (
+    default_tokenizer_name_1,
+    default_tokenizer_name_2,
+    default_user_input,
+    examples,
+)
+from playground_tokenizers import TokenizerFactory
+
+
+@lru_cache
+def run_tokenization(
+    text: str, tokenizer_name: str, color_num: int = 5, add_special_token: bool = False
+) -> tuple[list[tuple[str, str]], int, pd.DataFrame]:
+    """Tokenize an input text and return the tokens with their positions."""
+    logger.info(
+        "param="
+        + json.dumps(
+            {"text": text, "tokenizer_type": tokenizer_name}, ensure_ascii=False
+        )
+    )
+    pos_tokens = []
+    tokenizer = TokenizerFactory().get_tokenizer(tokenizer_name)
+    encoding = tokenizer.encode(text) if add_special_token else tokenizer.encode(text)
+    table = []
+
+    for idx, token_id in enumerate(encoding):
+        decoded_text = tokenizer.decode([token_id])
+        decoded_text = decoded_text.replace(
+            " ", "⋅"
+        )  # replace space with ⋅ for better visualization
+        pos_tokens.extend([(decoded_text, str(idx % color_num))])
+
+        try:
+            token = tokenizer.decode([token_id])[0]
+        except:
+            token = {v: k for k, v in tokenizer.get_vocab().items()}[token_id]
+
+        if isinstance(token, bytes):
+            try:
+                token_str = token.decode("utf-8")
+            except:
+                token_str = token.decode("utf-8", errors="ignore")
+                logger.error(
+                    f"{idx}: decode_error: {tokenizer_name}, {token} {token_str}"
+                )
+
+        elif isinstance(token, str):
+            token_str = token
+        else:
+            logger.error(
+                f"{idx}: wrong type for token {token_id} {type(token)} "
+                + json.dumps(
+                    {"text": text, "tokenizer_type": tokenizer_name}, ensure_ascii=False
+                )
+            )
+            token_str = token
+
+        table.append({"TokenID": token_id, "Text": decoded_text})
+
+    table_df = pd.DataFrame(table)
+    logger.info(f"tokenizer_type={tokenizer_name}, Tokens={table[:4]}")
+    return pos_tokens, len(encoding), table_df
+
+
+def tokenize(
+    text: str, tokenizer_name: str, color_num: int = 5
+) -> tuple[dict[Any, Any], pd.DataFrame]:
+    """Tokenize an input text."""
+    pos_tokens, num_tokens, table_df = run_tokenization(text, tokenizer_name, color_num)
+    return gr.update(value=pos_tokens, label=f"Tokens: {num_tokens}"), table_df
+
+
+def tokenize_pair(
+    text: str, tokenizer_name_1: str, tokenizer_name_2: str, color_num: int = 5
+):
+    """input_text.change."""
+    pos_tokens_1, table_df_1 = tokenize(
+        text=text, tokenizer_name=tokenizer_name_1, color_num=color_num
+    )
+    pos_tokens_2, table_df_2 = tokenize(
+        text=text, tokenizer_name=tokenizer_name_2, color_num=color_num
+    )
+    return pos_tokens_1, table_df_1, pos_tokens_2, table_df_2
+
+
+def on_load(url_params: str, request: gr.Request | None = None) -> tuple[str, str, str]:
+    """Function triggered on page load to get URL parameters."""
+    text = default_user_input
+    tokenizer_type_1 = default_tokenizer_name_1
+    tokenizer_type_2 = default_tokenizer_name_2
+    return text, tokenizer_type_1, tokenizer_type_2
+
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = JSON.stringify(Object.fromEntries(params));
+        return url_params;
+        }
+    """
+
+all_tokenizer_name = [
+    (config.name_display, config.name_or_path)
+    for config in TokenizerFactory().all_tokenizer_configs
+]
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        gr.Markdown("## Input Text")
+        dropdown_examples = gr.Dropdown(
+            sorted(examples.keys()),
+            value="Examples",
+            type="index",
+            allow_custom_value=True,
+            show_label=False,
+            container=False,
+            scale=0,
+            elem_classes="example-style",
+        )
+    user_input = gr.Textbox(
+        label="Input Text",
+        lines=5,
+        show_label=False,
+    )
+
+    with gr.Row():
+        with gr.Column(scale=6), gr.Group():
+            tokenizer_name_1 = gr.Dropdown(all_tokenizer_name, label="Tokenizer 1")
+
+        with gr.Column(scale=6), gr.Group():
+            tokenizer_name_2 = gr.Dropdown(all_tokenizer_name, label="Tokenizer 2")
+
+    with gr.Row():
+        with gr.Column():
+            output_text_1 = gr.Highlightedtext(
+                show_legend=False, show_inline_category=False
+            )
+        with gr.Column():
+            output_text_2 = gr.Highlightedtext(
+                show_legend=False, show_inline_category=False
+            )
+
+    with gr.Row():
+        output_table_1 = gr.Dataframe()
+        output_table_2 = gr.Dataframe()
+
+    tokenizer_name_1.change(
+        tokenize, [user_input, tokenizer_name_1], [output_text_1, output_table_1]
+    )
+
+    tokenizer_name_2.change(
+        tokenize, [user_input, tokenizer_name_2], [output_text_2, output_table_2]
+    )
+
+    user_input.change(
+        tokenize_pair,
+        [user_input, tokenizer_name_1, tokenizer_name_2],
+        [output_text_1, output_table_1, output_text_2, output_table_2],
+        show_api=False,
+    )
+
+    dropdown_examples.change(
+        lambda example_idx: (
+            examples[sorted(examples.keys())[example_idx]]["text"],
+            examples[sorted(examples.keys())[example_idx]]["tokenizer_1"],
+            examples[sorted(examples.keys())[example_idx]]["tokenizer_2"],
+        ),
+        dropdown_examples,
+        [user_input, tokenizer_name_1, tokenizer_name_2],
+        show_api=False,
+    )
+
+    demo.load(
+        fn=on_load,
+        inputs=[user_input],
+        outputs=[user_input, tokenizer_name_1, tokenizer_name_2],
+        js=get_window_url_params,
+        show_api=False,
+    )
+
+if __name__ == "__main__":
+    demo.launch(share=True)

playground_examples.py

@@ -0,0 +1,42 @@
+default_user_input = """Replace this text in the input field to see how tokenization works."""
+default_tokenizer_name_1 = "openai/gpt-4o"
+default_tokenizer_name_2 = "Qwen/Qwen2.5-72B"
+
+
+number_example = """127+677=804
+127 + 677 = 804\n
+1275+6773 = 8041
+1275 + 6773 = 8048"""
+
+code_example = """for i in range(1, 101):
+    if i % 3 == 0 and i % 5 == 0:
+        print("FizzBuzz")
+    elif i % 3 == 0:
+        print("Fizz")
+    elif i % 5 == 0:
+        print("Buzz")
+    else:
+        print(i)
+"""
+
+spelling_example = """How do you spell "accommodate"?
+How many letters are in the word "accommodate"?
+How many r's are in the word strawberry?"""
+
+examples = {
+    "number": {
+        "text": number_example,
+        "tokenizer_1": default_tokenizer_name_1,
+        "tokenizer_2": default_tokenizer_name_2,
+    },
+    "code": {
+        "text": code_example,
+        "tokenizer_1": default_tokenizer_name_1,
+        "tokenizer_2": default_tokenizer_name_2,
+    },
+    "spelling": {
+        "text": spelling_example,
+        "tokenizer_1": default_tokenizer_name_1,
+        "tokenizer_2": default_tokenizer_name_2,
+    },
+}

playground_tokenizers.py

@@ -0,0 +1,175 @@
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from typing import Any
+
+import tiktoken
+from loguru import logger
+from transformers import AutoTokenizer
+
+
+class TokenizerImpl(Enum):
+    BertTokenizer = "wordpiece.BertTokenizer"
+    ByteLevelBPETokenizer = "byte_level_bpe"
+    SentencePieceBPETokenizer = "sentencepiece_bpe"
+
+    SentencePiece = auto()
+    byte_level_bpe = auto()
+
+    TikToken = auto()
+
+
+@dataclass
+class TokenizerConfig:
+    """Tokenizer Configuration"""
+
+    name_or_path: str
+    name_display: str | None = None
+    impl: TokenizerImpl | None = None
+    org: str | None = None
+    link: str | None = None
+    desc: str | None = None
+    meta: str | None = None
+    level: str | None = None
+    lang: str | None = None
+    init_kwargs: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        if self.link is None:
+            self.link = "https://huggingface.co/" + self.name_or_path
+        if self.name_display is None:
+            self.name_display = self.name_or_path
+
+    @classmethod
+    def init_from_json_file(cls, json_filepath: str) -> "TokenizerConfig":
+        pass
+
+    def __eq__(self, other):
+        if isinstance(other, self.__class__):
+            return self.__dict__ == other.__dict__
+        else:
+            return False
+
+    def __hash__(self):
+        return hash(self.name_or_path)
+
+
+tokenizer_configs = [
+    TokenizerConfig(
+        "google-bert/bert-base-uncased",
+        impl=TokenizerImpl.BertTokenizer,
+        org="Google",
+        desc="first add whitespace around any CJK character, then perform wordpiece tokenization.",
+    ),
+    TokenizerConfig(
+        "google-bert/bert-base-multilingual-uncased",
+        impl=TokenizerImpl.BertTokenizer,
+        org="Google",
+    ),
+    TokenizerConfig(
+        "openai-community/gpt2", impl=TokenizerImpl.SentencePiece, org="OpenAI"
+    ),
+    TokenizerConfig(
+        "EleutherAI/gpt-neox-20b", impl=TokenizerImpl.SentencePiece, org="EleutherAI"
+    ),
+    TokenizerConfig(
+        "Qwen/Qwen1.5-14B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
+    ),
+    TokenizerConfig(
+        "Qwen/Qwen2.5-72B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
+    ),
+    TokenizerConfig(
+        "google-t5/t5-large",
+        name_display="google-t5/t5",
+        impl=TokenizerImpl.SentencePiece,
+        org="Google",
+    ),
+    TokenizerConfig("CohereForAI/aya-101", org="Cohere For AI"),
+    TokenizerConfig(
+        "meta-llama/Llama-3.2-3B-Instruct", impl=TokenizerImpl.SentencePiece, org="Meta"
+    ),
+    TokenizerConfig(
+        "openai/gpt-4o",
+        impl=TokenizerImpl.TikToken,
+        org="OpenAI",
+        link="https://github.com/openai/tiktoken",
+    ),
+    TokenizerConfig("google/mt5-large", org="Google"),
+    TokenizerConfig("deepseek-ai/deepseek-coder-33b-instruct", org="DeepSeek"),
+    TokenizerConfig("deepseek-ai/DeepSeek-V3", org="DeepSeek"),
+]
+
+assert len(set([config.name_display for config in tokenizer_configs])) == len(
+    tokenizer_configs
+)
+assert len(set([config.name_or_path for config in tokenizer_configs])) == len(
+    tokenizer_configs
+)
+assert len(
+    set([config.name_or_path.split("/")[-1] for config in tokenizer_configs])
+) == len(tokenizer_configs)
+
+
+class TokenizerFactory:
+    def __init__(self):
+        self.all_tokenizer_configs = sorted(
+            tokenizer_configs, key=lambda k: k.name_display
+        )
+        self.all_tokenizer_names = [
+            config.name_or_path for config in self.all_tokenizer_configs
+        ]
+        self.name_to_config_list = [
+            {config.name_or_path: config for config in self.all_tokenizer_configs},
+            {config.name_display: config for config in self.all_tokenizer_configs},
+            {
+                config.name_display.split("/")[-1]: config
+                for config in self.all_tokenizer_configs
+            },
+        ]
+        self.tokenizer_cache = {}
+
+    def get_tokenizer_config(self, tokenizer_name: str) -> TokenizerConfig | None:
+        for name_to_config in self.name_to_config_list:
+            if tokenizer_name in name_to_config:
+                return name_to_config[tokenizer_name]
+        return None
+
+    def get_tokenizer(self, tokenizer_name: str) -> AutoTokenizer:
+        """Get the tokenizer by its name, loading it if not already cached."""
+        tokenizer_config = self.get_tokenizer_config(tokenizer_name)
+
+        if tokenizer_config in self.tokenizer_cache:
+            return self.tokenizer_cache[tokenizer_config]
+
+        tokenizer = self.load_tokenizer(tokenizer_config)
+
+        self.tokenizer_cache[tokenizer_config] = tokenizer
+        return tokenizer
+
+    def get_name_with_hyperlink(self, tokenizer_name: str) -> str:
+        def model_hyperlink(link, model_name):
+            model_name = model_name
+            return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+        tokenizer_config = self.get_tokenizer_config(tokenizer_name)
+        return model_hyperlink(
+            tokenizer_config.link, tokenizer_config.name_display.split("/")[-1]
+        )
+
+    def load_tokenizer(self, tokenizer_config):
+        if tokenizer_config == None:
+            print("dd")
+        logger.info(f"loading tokenizer {tokenizer_config.name_or_path}")
+        if (
+            tokenizer_config.impl == TokenizerImpl.TikToken
+            and "openai" in tokenizer_config.name_or_path
+        ):
+            tokenizer = tiktoken.encoding_for_model(
+                tokenizer_config.name_or_path.replace("openai/", "")
+            )
+        else:
+            tokenizer = AutoTokenizer.from_pretrained(
+                tokenizer_config.name_or_path,
+                trust_remote_code=True,
+                **tokenizer_config.init_kwargs,
+            )
+        return tokenizer

requirements.txt

@@ -0,0 +1,13 @@
+gradio>=4.38.1
+transformers>4.40.0
+sentencepiece
+tiktoken
+icetk
+torch
+nltk
+boto3
+protobuf==4.25.3
+ai2-olmo
+ipadic
+fugashi
+datasets