Add application file

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Athens NLP Summer School
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,12 +1 @@
----
-title: Test
-emoji: 📉
-colorFrom: blue
-colorTo: red
-sdk: gradio
-sdk_version: 5.34.2
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# tokenization_playground

app.py

@@ -0,0 +1,24 @@
+import os
+
+import gradio as gr
+from huggingface_hub import login
+from playground_app import demo as playground_tab
+
+auth_token = os.environ.get("HF_TOKEN", None)
+if auth_token:
+    login(token=auth_token)
+
+
+title = """
+<div align="center">
+    <span>Tokenization Playground</span>
+</div>
+"""
+
+with gr.Blocks() as demo:
+    _ = gr.HTML(f"<h1 style='text-align: center; margin-bottom: 1rem'>{title}</h1>")
+    _ = playground_tab.render()
+
+if __name__ == "__main__":
+    # demo.launch()
+    demo.launch(share=True)

character_util.py

@@ -0,0 +1,178 @@
+import json
+import os
+from pathlib import Path
+from typing import Literal
+
+import numpy as np
+import pandas as pd
+from utils.lang_util import detect_language_by_unicode, language_ranges
+from utils.log_util import logger
+from utils.text_util import contains_digit, get_space_count
+from vocab import tokenizer_factory
+
+CURRENT_DIR = Path.parent(Path.resolve(__file__))
+
+cache = {}
+default_columns = ["digit", "zh"]
+
+
+def text_to_unicode(text: str) -> str:
+    """Convert text to unicode representation."""
+    return "".join(rf"\u{ord(character):04X}" for character in text)
+
+
+def calculate_dist(token_lens: list[int]) -> str:
+    """Calculate the distribution of token lengths."""
+    if not token_lens:
+        return "-"
+    return f"{min(token_lens)},{round(np.median(token_lens))},{max(token_lens)}"
+
+
+def iter_vocab(
+    tokenizer_name: str,
+    from_cache: bool = True,
+    cache_dir: str = "stats",
+) -> pd.DataFrame | dict:
+    """:param tokenizer_name:
+    :param from_cache:
+    :param cache_dir:
+    :return:
+    """
+    tokenizer_config = tokenizer_factory.get_tokenizer_config(tokenizer_name)
+
+    cache_dir = os.path.join(CURRENT_DIR, cache_dir)
+    os.makedirs(cache_dir, exist_ok=True)
+
+    # load from cache
+    cache_path = os.path.join(cache_dir, "character_stats.json")
+    if not cache and os.path.exists(cache_path):
+        with open(cache_path, encoding="utf-8") as f_tmp:
+            cache.update(json.load(f_tmp))
+    if from_cache and tokenizer_name in cache:
+        # logger.info(f"load {tokenizer_config.name_or_path} from cache")
+        return cache[tokenizer_name]
+
+    tokenizer = tokenizer_factory.get_tokenizer(tokenizer_name)
+
+    tokens_by_lang = {lang[1]: [] for lang in language_ranges}
+    digit_tokens = []
+    space_tokens = []
+    byte_tokens = []
+
+    buffer = []
+    for token_id in range(tokenizer.vocab_size):
+        # for token_id in tokenizer.get_vocab():
+        # for token_id in range(len(tokenizer)):
+        decode_str = tokenizer.decode([token_id], skip_special_tokens=False)
+        token = tokenizer.convert_ids_to_tokens([token_id], skip_special_tokens=False)[0]
+        tags = []
+        if token is None:  # 有些词典有空的id（不连续）
+            continue
+        if isinstance(token, bytes):
+            token = token.decode("utf-8", errors="ignore")
+
+        if hasattr(tokenizer, "sp_model") and tokenizer.sp_model.is_byte(token_id):
+            tags.append("is_byte")
+            byte_tokens.append(token)
+
+        language_tags = detect_language_by_unicode(decode_str)
+        for language in language_tags:
+            tokens_by_lang[language[1]].append(decode_str)
+
+        if contains_digit(decode_str):
+            tags.append("digit")
+            digit_tokens.append(decode_str)
+
+        space_count = get_space_count(decode_str)
+        if space_count > 0:
+            space_tokens.append(decode_str)
+
+        buffer.append(
+            json.dumps(
+                {
+                    "id": token_id,
+                    "token": token,
+                    "token_decode": decode_str,
+                    "token_dumps": json.dumps(token),
+                    "token_unicode": text_to_unicode(token),
+                    "token_len": len(decode_str),
+                },
+                ensure_ascii=False,
+            )
+            + "\n"
+        )
+
+    result = {
+        "tokenizer": tokenizer_factory.get_name_with_hyperlink(tokenizer_name),
+        "organization": tokenizer_config.org,
+        "vocab_size": len(tokenizer),
+        "num(digit)": len(digit_tokens),
+        "len(digit)": calculate_dist([len(token) for token in digit_tokens]),
+        "num(space)": len(space_tokens),
+        "len(space)": calculate_dist([len(token) for token in space_tokens]),
+    }
+
+    for lang, tokens in tokens_by_lang.items():
+        result[f"num({lang})"] = len(tokens)
+        result["len(" + lang + ")"] = calculate_dist([len(token) for token in tokens])
+
+    out_path = os.path.join(
+        cache_dir, f"iter_vocab/{tokenizer_name.replace('/', '_')}.vocab.jsonl"
+    )
+    with open(out_path, "w", encoding="utf-8") as f_out:
+        for line in buffer:
+            f_out.write(line)
+    len_before = len(cache)
+    cache[tokenizer_name] = result
+    len_after = len(cache)
+    logger.info(f"saving {tokenizer_name} to memory and file cache: {len_before}->{len_after}")
+    with open(cache_path, "w", encoding="utf-8") as f_out:
+        f_out.write(json.dumps(cache, ensure_ascii=False, indent=2))
+    return result
+
+
+def to_dataframe(stats: dict[str, Any], columns: list[str]) -> pd.DataFrame:
+    table = []
+    for stat in stats.values():
+        filtered_stat = {}
+        for k, v in stat.items():
+            if not k.startswith("num") and not k.startswith("len"):
+                filtered_stat[k] = v
+            if any(column in k for column in columns):
+                k = k.replace("ja-kana", "kana")
+                filtered_stat[k] = v
+        table.append(filtered_stat)
+    return pd.DataFrame(table)
+
+
+def get_character_table(
+    tokenizer_filter: str | None = None,
+    columns: list | None = None,
+    return_type: Literal["dict", "dataframe"] | None = "dataframe",
+) -> pd.DataFrame | dict:
+    logger.info(f"columns: {columns}, tokenizer_filter: {tokenizer_filter}")
+    stats = {}
+    if columns is None:
+        columns = default_columns
+    if tokenizer_filter is not None:
+        tokenizer_names = [
+            tokenizer_config.name_or_path
+            for tokenizer_config in tokenizer_factory.all_tokenizer_configs
+            if tokenizer_filter.lower() in tokenizer_config.name_or_path.lower()
+        ]
+    else:
+        tokenizer_names = tokenizer_factory.all_tokenizer_names
+
+    for tokenizer_name in tokenizer_names:
+        stat = iter_vocab(tokenizer_name)
+        stats[tokenizer_name] = stat
+
+    if return_type == "dataframe":
+        stats = to_dataframe(stats, columns)
+    return stats
+
+
+if __name__ == "__main__":
+    # aa = get_character_table(tokenizer_filter="baichuan")
+    df = get_character_table()
+    logger.info(f"\n{df.to_markdown(index=False)}")

playground_app.py

@@ -0,0 +1,91 @@
+import gradio as gr
+from playground_examples import examples
+from playground_util import on_load, tokenize, tokenize_pair
+from vocab import tokenizer_factory
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = JSON.stringify(Object.fromEntries(params));
+        return url_params;
+        }
+    """
+
+all_tokenizer_name = [
+    (config.name_display, config.name_or_path)
+    for config in tokenizer_factory.all_tokenizer_configs
+]
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        gr.Markdown("## Input Text")
+        dropdown_examples = gr.Dropdown(
+            sorted(examples.keys()),
+            value="Examples",
+            type="index",
+            allow_custom_value=True,
+            show_label=False,
+            container=False,
+            scale=0,
+            elem_classes="example-style",
+        )
+    user_input = gr.Textbox(
+        label="Input Text",
+        lines=5,
+        show_label=False,
+    )
+
+    with gr.Row():
+        with gr.Column(scale=6), gr.Group():
+            tokenizer_name_1 = gr.Dropdown(all_tokenizer_name, label="Tokenizer 1")
+
+        with gr.Column(scale=6), gr.Group():
+            tokenizer_name_2 = gr.Dropdown(all_tokenizer_name, label="Tokenizer 2")
+
+    with gr.Row():
+        # dynamic change label
+        with gr.Column():
+            output_text_1 = gr.Highlightedtext(show_legend=False, show_inline_category=False)
+        with gr.Column():
+            output_text_2 = gr.Highlightedtext(show_legend=False, show_inline_category=False)
+
+    with gr.Row():
+        output_table_1 = gr.Dataframe()
+        output_table_2 = gr.Dataframe()
+
+    tokenizer_name_1.change(
+        tokenize, [user_input, tokenizer_name_1], [output_text_1, output_table_1]
+    )
+
+    tokenizer_name_2.change(
+        tokenize, [user_input, tokenizer_name_2], [output_text_2, output_table_2]
+    )
+
+    user_input.change(
+        tokenize_pair,
+        [user_input, tokenizer_name_1, tokenizer_name_2],
+        [output_text_1, output_table_1, output_text_2, output_table_2],
+        show_api=False,
+    )
+
+    dropdown_examples.change(
+        lambda example_idx: (
+            examples[sorted(examples.keys())[example_idx]]["text"],
+            examples[sorted(examples.keys())[example_idx]]["tokenizer_1"],
+            examples[sorted(examples.keys())[example_idx]]["tokenizer_2"],
+        ),
+        dropdown_examples,
+        [user_input, tokenizer_name_1, tokenizer_name_2],
+        show_api=False,
+    )
+
+    demo.load(
+        fn=on_load,
+        inputs=[user_input],
+        outputs=[user_input, tokenizer_name_1, tokenizer_name_2],
+        js=get_window_url_params,
+        show_api=False,
+    )
+
+if __name__ == "__main__":
+    demo.launch(share=True)

playground_examples.py

@@ -0,0 +1,42 @@
+default_user_input = """Replace this text in the input field to see how tokenization works."""
+default_tokenizer_name_1 = "openai/gpt-4o"
+default_tokenizer_name_2 = "Qwen/Qwen2.5-72B"
+
+
+number_example = """127+677=804
+127 + 677 = 804\n
+1275+6773 = 8041
+1275 + 6773 = 8048"""
+
+code_example = """for i in range(1, 101):
+    if i % 3 == 0 and i % 5 == 0:
+        print("FizzBuzz")
+    elif i % 3 == 0:
+        print("Fizz")
+    elif i % 5 == 0:
+        print("Buzz")
+    else:
+        print(i)
+"""
+
+spelling_example = """How do you spell "accommodate"?
+How many letters are in the word "accommodate"?
+How many r's are in the word strawberry?"""
+
+examples = {
+    "number": {
+        "text": number_example,
+        "tokenizer_1": default_tokenizer_name_1,
+        "tokenizer_2": default_tokenizer_name_2,
+    },
+    "code": {
+        "text": code_example,
+        "tokenizer_1": default_tokenizer_name_1,
+        "tokenizer_2": default_tokenizer_name_2,
+    },
+    "spelling": {
+        "text": spelling_example,
+        "tokenizer_1": default_tokenizer_name_1,
+        "tokenizer_2": default_tokenizer_name_2,
+    },
+}

playground_util.py

@@ -0,0 +1,107 @@
+import json
+from functools import lru_cache
+from typing import Any
+
+import gradio as gr
+import pandas as pd
+from playground_examples import (
+    default_tokenizer_name_1,
+    default_tokenizer_name_2,
+    default_user_input,
+)
+from utils.i18n_util import get_lang
+from utils.log_util import logger
+from vocab import tokenizer_factory
+
+
+@lru_cache
+def _tokenize(text: str, tokenizer_name: str, color_num: int = 5, add_special_token: bool = False):
+    logger.info(
+        "param=" + json.dumps({"text": text, "tokenizer_type": tokenizer_name}, ensure_ascii=False)
+    )
+    pos_tokens = []
+    tokenizer = tokenizer_factory.get_tokenizer(tokenizer_name)
+    encoding = tokenizer.encode(text) if add_special_token else tokenizer.encode(text)
+    table = []
+
+    for idx, token_id in enumerate(encoding):
+        decoded_text = tokenizer.decode([token_id])
+        decoded_text = decoded_text.replace(
+            " ", "⋅"
+        )  # replace space with ⋅ for better visualization
+        pos_tokens.extend([(decoded_text, str(idx % color_num))])
+
+        try:
+            token = tokenizer.decode([token_id])[0]
+        except:
+            token = {v: k for k, v in tokenizer.get_vocab().items()}[token_id]
+
+        if isinstance(token, bytes):
+            try:
+                token_str = token.decode("utf-8")
+            except:
+                token_str = token.decode("utf-8", errors="ignore")
+                logger.error(
+                    f"{idx}: decode_error: "
+                    + json.dumps(  # gpt_35_turbo 经常有token会decode error，这里用来记录一下
+                        {
+                            "tokenizer_type": tokenizer_name,
+                            "token": str(token),
+                            "token_str": token_str,
+                        },
+                        ensure_ascii=False,
+                    )
+                )
+
+            # json_dumps = json.dumps(token_str)
+        elif isinstance(token, str):
+            token_str = token
+        else:
+            logger.error(
+                f"{idx}: wrong type for token {token_id} {type(token)} "
+                + json.dumps({"text": text, "tokenizer_type": tokenizer_name}, ensure_ascii=False)
+            )
+            token_str = token
+
+        table.append({"TokenID": token_id, "Text": decoded_text})
+
+    table_df = pd.DataFrame(table)
+    logger.info(f"tokenizer_type={tokenizer_name}, Tokens={table[:4]}")
+    return pos_tokens, len(encoding), table_df
+
+
+def tokenize(
+    text: str, tokenizer_name: str, color_num: int = 5
+) -> tuple[dict[Any, Any], pd.DataFrame]:
+    """Tokenize an input text."""
+    pos_tokens, num_tokens, table_df = _tokenize(text, tokenizer_name, color_num)
+    return gr.update(value=pos_tokens, label=f"Tokens: {num_tokens}"), table_df
+
+
+def tokenize_pair(text, tokenizer_type_1, tokenizer_type_2, color_num: int = 5):
+    """input_text.change."""
+    pos_tokens_1, table_df_1 = tokenize(text, tokenizer_type_1, color_num)
+    pos_tokens_2, table_df_2 = tokenize(text, tokenizer_type_2, color_num)
+    return pos_tokens_1, table_df_1, pos_tokens_2, table_df_2
+
+
+def on_load(url_params: str, request: gr.Request = None) -> tuple[str, str, str]:
+    """Function triggered on page load to get URL parameters."""
+    text = default_user_input
+    tokenizer_type_1 = default_tokenizer_name_1
+    tokenizer_type_2 = default_tokenizer_name_2
+    try:
+        url_params_dict = json.loads(url_params)
+    except json.JSONDecodeError:
+        url_params_dict = {}
+
+    if request:
+        lang, _ = get_lang(request)
+        logger.info(str(request.headers))
+        client_ip = request.client.host
+
+        tokenizer_type_1 = url_params_dict.get("tokenizer1", default_tokenizer_name_1)
+        tokenizer_type_2 = url_params_dict.get("tokenizer2", default_tokenizer_name_2)
+        text = url_params_dict.get("text", default_user_input)
+        logger.info(f"client_ip: {client_ip}; lang: {lang} params: {url_params}")
+    return text, tokenizer_type_1, tokenizer_type_2

requirements.txt

@@ -0,0 +1,13 @@
+gradio>=4.38.1
+transformers>4.40.0
+sentencepiece
+tiktoken
+icetk
+torch
+nltk
+boto3
+protobuf==4.25.3
+ai2-olmo
+ipadic
+fugashi
+datasets

utils/i18n_util.py

@@ -0,0 +1,26 @@
+import gradio as gr
+
+
+def get_lang(request: gr.Request):
+    """
+    'accept-language', b'zh,en;q=0.9,zh-CN;q=0.8')
+    """
+    accept_language = None
+    langs = []
+    try:
+        accept_language = request.headers["Accept-Language"]
+        for lang in accept_language.split(",")[:5]:
+            lang = lang.lower()
+            if lang.startswith("en"):
+                langs.append("en")
+            elif lang.startswith("es"):
+                langs.append("es")
+            elif lang.startswith("zh"):
+                langs.append("zh")
+            elif lang.startswith("fr"):
+                langs.append("fr")
+            elif lang.startswith("de"):
+                langs.append("de")
+    except Exception as e:
+        print(e)
+    return accept_language, langs

utils/lang_util.py

@@ -0,0 +1,89 @@
+"""
+这个detect_language函数通过定义一系列语言字符的Unicode范围，然后使用regex包来检查输入字符串是否包含这些范围内的字符，
+从而尝试确定字符串可能使用的语言。函数返回一个列表，包含所有匹配的语言名称；如果没有检测到已定义范围内的字符，则返回['Unknown']。
+
+请注意，由于某些语言（如中文和日文）共享字符集的部分范围，这可能导致某些字符串被识别为多种语言。
+此外，Latin范围非常广泛，几乎包括了所有西方语言的基本字母，因此可能需要更细致的逻辑来区分使用拉丁字母的具体语言。
+
+
+通过检查特定的字母和重音符号来区分一些使用拉丁字母的语言。
+然而，需要强调的是，这种方法的准确性受限于所选语言特征的全面性和独特性。
+例如，English的检测范围仅限于基本的A-Z字母，这可能导致它与其他使用相同字母集的语言重叠。
+此外，有些语言（如法语和西班牙语）在某些情况下可能共享特定的重音符号，这可能导致一个字符串被错误地识别为多种语言。
+
+## common language
+English | 简体中文 | 繁體中文 | 한국어 | Español | 日本語 | हिन्दी | Русский | Рortuguês | తెలుగు | Français | Deutsch | Tiếng Việt |
+"""
+
+import re
+from typing import List
+
+# 由于大部分是'latin'，所以就不统计了。
+common_lang = ["Chinese", "Japanese-Kana", "Korean", "Arabic", "number"]
+
+# Unicode range of different language
+language_ranges = {
+    (
+        "Arabic",
+        "ar",
+    ): r"[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF]",
+    # 'CJK'  https://en.wikipedia.org/wiki/CJK_Unified_Ideographs
+    ("Chinese", "zh"): r"[\u4e00-\u9fff]",
+    ("Japanese", "ja"): r"[\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF]",
+    # https://stackoverflow.com/questions/19899554/unicode-range-for-japanese
+    # Kana type refers to Japanese hiragana and katakana characters that represent phonetic sounds in the Japanese language.
+    (
+        "Japanese-Kana",
+        "ja-kana",
+    ): r"[\u3040-\u309F\u30A0-\u30FF]",  # Hiragana  & Katakana
+    ("Korean", "ko"): r"[\uac00-\ud7a3]",
+    # 拉丁字母系列
+    # ('Latin', 'la'): r'[\u0000-\u007F\u0080-\u00FF]',
+    # ('English', 'en'): r'[A-Za-z]',  # 这可能会与其他使用基本拉丁字母的语言重叠
+    # ('French', 'fr'): r'[\u00C0-\u00FF]',
+    # ('German', 'de'): r'[\u00C4\u00D6\u00DC\u00E4\u00F6\u00FC\u00DF]',
+    # ('Spanish-特有'): r'[\u00C1\u00E1\u00C9\u00E9\u00CD\u00ED\u00D3\u00F3\u00DA\u00FA\u00D1\u00F1\u00FC]',  # 西班牙语特有字符集合
+    # 斯拉夫语系列
+    # ('Cyrillic', ''): r'[\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]',
+    #
+    # 'Greek': r'[\u0370-\u03FF\u1F00-\u1FFF]',   # 希腊字母
+    # 'Hebrew': r'[\u0590-\u05FF\uFB1D-\uFB4F]',  # 希伯来语
+}
+
+
+def detect_language_by_unicode(text: str) -> List:
+    """
+    :param text:
+    :return:
+    """
+    detected_languages = []
+    for language, pattern in language_ranges.items():
+        if re.search(pattern, text):
+            detected_languages.append(language)
+
+    return detected_languages
+
+
+if __name__ == "__main__":
+    # 测试函数
+    test_strings = {
+        # 拉丁语系
+        "Hello, world!": "English/Latin",
+        "Hola": "Spanish",
+        "Bonjour": "French",
+        "Guten Tag": "German",
+        "Empieza donde estás. ": "Spanish",
+        # CJK
+        "你好": "Chinese",
+        "こんにちは": "Japanese",
+        "안녕하세요": "Korean",
+        # 其他
+        "Привет": "Russian/Cyrillic",
+        "مرحبا": "Arabic",
+    }
+
+    for s, expected in test_strings.items():
+        # print(f"'{s}' === Detected lang: {detect_language(s)} === Expected: {expected}")
+        print(
+            f"'{s}'\nDetected lang: {detect_language_by_unicode(s)}\nExpected lang: {expected}"
+        )

utils/log_util.py

@@ -0,0 +1,10 @@
+import logging
+
+logging.basicConfig(
+    format="[%(asctime)s] [%(levelname)s] [%(process)d:%(thread)d] [%(filename)s:%(lineno)d:%(funcName)s] %(message)s",
+    level=logging.INFO,
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)

utils/oov_util.py

@@ -0,0 +1,122 @@
+import json
+
+from vocab import TokenizerImpl, all_tokenizer_config, load_tokenizer
+
+text = (
+    "hello; Замглавы управления развития; 특히 주소 15~17번 홀에선 3연속;"
+    " 確実に春が近づいてること;  a közoktatással? _ Belföld;"
+    " pumë, i vjetër, vjeç; ئەردوغان ۋە قىرغىزىستان ;"
+    " निम्न में से कौन सा हारडवेयर; ተለዋዋጭ የግድግዳ ; Дзейныя асобы:;"
+    " « અમરેલીનાં મહિલા વિકાસ; 🦙❤❥웃유♋☮✊;"
+    "װיקיװערטערבוך "
+)
+whitespace = "\t   \n\n\r  "
+bytes = b"\x00\x01\x02\x03\x04".decode("utf-8")
+
+text += whitespace
+
+
+def get_unk(tokenizer_config):
+    tokenizer = load_tokenizer(tokenizer_config)
+    if hasattr(tokenizer, "unk_token"):
+        return f"{tokenizer.unk_token}, {tokenizer.unk_token_id}"
+    else:
+        return "unk_token not found"
+
+
+# def infer_tokenizer_impl(tokenizer_config):
+def infer_tokenizer_type(tokenizer_config):
+    tokenizer = load_tokenizer(tokenizer_config)
+    if tokenizer_config.impl == TokenizerImpl.TikToken:
+        return "tiktoken"
+    if hasattr(tokenizer, "backend_tokenizer"):
+        return str(
+            type(tokenizer.backend_tokenizer.model)
+        )  # type(tokenizer._tokenizer.model))
+    # orion: sp_model.Load(vocab_file)，继承 PreTrainedTokenizer
+    elif hasattr(tokenizer, "sp_model"):  # 基于 sentencepiece 包
+        # for i in range(tokenizer.sp_model.piece_size()):
+        #     if tokenizer.sp_model.is_byte(i):
+        #         print("")
+        return f"sp_model, byte_num: {sum([tokenizer.sp_model.is_byte(i) for i in range(tokenizer.sp_model.piece_size())])}"
+
+    # sp.Load(model_path)  ，并且包括image_tokenizer
+    elif "glm-" in tokenizer_config.name_or_path:
+        return f"byte_num: {sum([tokenizer.sp_tokenizer.text_tokenizer.sp.is_byte(i) for i in range(tokenizer.sp_tokenizer.text_tokenizer.sp.piece_size())])}"
+    # sp.Load(model_path)  ，没有image_tokenizer
+    elif (
+        "glm2-" in tokenizer_config.name_or_path
+        or "glm3-" in tokenizer_config.name_or_path
+        or "CharacterGLM-6B" in tokenizer_config.name_or_path
+    ):
+        return f"byte_num: {sum([tokenizer.tokenizer.sp_model.is_byte(i) for i in range(tokenizer.tokenizer.sp_model.piece_size())])}"
+    elif (
+        "abeja/gpt-neox-japanese-2.7b" == tokenizer_config.name_or_path
+    ):  # 支持 byte-level，解决oov问题
+        return "japanese-bpe: https://github.com/tanreinama/Japanese-BPEEncoder_V2"
+    # bert-base-japanese： 特殊的地方在于 "word_tokenizer_type": "mecab"，见 https://huggingface.co/tohoku-nlp/bert-base-japanese/blob/main/tokenizer_config.json
+    elif "bert-base-japanese" in tokenizer_config.name_or_path:
+        return (
+            "wordpiece.MecabTokenizer, 支持byte-level https://taku910.github.io/mecab/"
+        )
+    elif "moss" in tokenizer_config.name_or_path:
+        return "应该是 sentencepiece.byte_bpe,待确认"
+    elif "byt5" in tokenizer_config.name_or_path:
+        return "未知，待定"
+    else:
+        print("catch", tokenizer_config.name_or_path)
+        raise "error"
+
+
+def test_lossless(tokenizer_config):
+    """
+    xlm-roberta-base 为什么oov这么少？是因为有 byte吗？
+    :param tokenizer_config:
+    :return:
+    """
+    tokenizer = load_tokenizer(tokenizer_config)
+    encoding = tokenizer.encode(text, add_special_tokens=False)
+    decoding = tokenizer.decode(encoding)
+
+    if text in decoding:
+        # print(tokenizer_config.name, tokenizer_config.impl, "lossless: true")
+        pass
+    else:
+        unk_count = sum(
+            [1 for token_id in encoding if token_id == tokenizer.unk_token_id]
+        )
+        oov_tokens = []
+        # if tokenizer_config.impl == TokenizerImpl.SentencePiece:
+        #     print(sum([tokenizer.is_byte(i) for i in range(tokenizer.piece_size())]))
+
+        print("#######" * 5)
+        print(
+            f"{tokenizer_config.name_or_path}, {infer_tokenizer_type(tokenizer_config)}\n"
+            f"lossless: false; unk_token: {get_unk(tokenizer_config)},"
+            f" unk_ratio: {unk_count/len(encoding):.4f}; oov: []"
+        )
+        for i in range(len(text)):
+            if text[i] != decoding[i]:
+                # print(f"text[{i}]     = {str(bytes(text[i:], 'utf-8'))}\n"
+                #       f"decoding[{i}] = {str(bytes(decoding[i:], 'utf-8'))}")
+                print(
+                    f"text[{i}]     = {json.dumps(text[i:], ensure_ascii=False)}, \n"
+                    f"decoding[{i}] = {json.dumps(decoding[i:], ensure_ascii=False)}"
+                )
+
+                break
+
+
+for config in all_tokenizer_config:
+    # if "xlm-roberta-base" in config.name:
+    # if "xlm-roberta-base" in config.name:
+    # if "chatglm3-6b" in config.name:
+    # if "bert-base-japanese" in config.name:
+    # if "moss" in config.name:
+    # if "byt5" in config.name:
+    if "baichuan" in config.name_or_path:
+        # if "CharacterGLM-6B" in config.name:
+        # if "fastchat-t5" in config.name:  # 报错 pyo3_runtime.PanicException: AddedVocabulary bad split
+        # if True:
+        # test_unk(config)
+        test_lossless(config)

utils/text_util.py

@@ -0,0 +1,47 @@
+"""
+char_
+"""
+
+
+def detect_lang_from_unicode():
+    pass
+
+
+def is_digit_char(uchar):
+    return uchar in "0123456789"
+
+
+def contains_digit(text):
+    return any(is_digit_char(ch) for ch in text)
+
+
+def get_digit_count(text):
+    pass
+
+
+def is_all_digit(text):
+    return all(is_digit_char(char) for char in text)
+
+
+def get_digit_count(text):
+    digit_count = 0
+    for char in text:
+        if char in "0123456789":
+            digit_count += 1
+    return digit_count
+
+
+def has_space(text):
+    pass
+
+
+def is_all_space(text):
+    pass
+
+
+def get_space_count(text):
+    space_count = 0
+    for char in text:
+        if len(char.strip()) == 0:
+            space_count += 1
+    return space_count

vocab.py

@@ -0,0 +1,754 @@
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from typing import Any, Dict
+
+import tiktoken
+from transformers import AutoTokenizer
+from utils.log_util import logger
+
+"""Interface:
+# https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py
+
+tokenizer.encode -> List[int]: Converts a string to a sequence of ids (integer)
+tokenizer.decode
+    tokenizer.convert_tokens_to_string   # gpt4 没有这个方法
+tokenizer.convert_ids_to_tokens
+tokenizer.tokenize -> List[str]:  Converts a string into a sequence of tokens ->
+
+
+tokenizer.parent = ""
+tokenizer.vocab_size   
+tokenizer.get_vocab()   # gpt-neox-20b, llama
+tokenizer.type = TokenizerType.ByteBPE.name
+tokenizer.implementation = TokenizerImpl.SentencePiece.name   # https://github.com/facebookresearch/llama/blob/main/llama/tokenizer.py
+  "HFGPT2Tokenizer", "HFTokenizer", "GPT2BPETokenizer", "CharLevelTokenizer", "TiktokenTokenizer", "SPMTokenizer", https://github.com/EleutherAI/gpt-neox/blob/main/tools/preprocess_data.py
+
+    
+tokenizer.comments = "split all numbers into individual digits, " \
+                     "and fallback to bytes to decompose unknown UTF-8 characters"
+
+tokenizer.all_special_tokens  # baichuan
+tokenizer.special_tokens_set   # gpt3.5_turbo
+tokenizer.special_tokens_map   
+"""
+
+
+class TokenizerImpl(Enum):
+    """
+    - https://github.com/huggingface/tokenizers/blob/main/bindings/python/py_src/tokenizers/implementations/__init__.py
+    - https://huggingface.co/docs/transformers/tokenizer_summary
+    - https://github.com/EleutherAI/gpt-neox/blob/main/megatron/tokenizer/tokenizer.py
+
+    ## google/BertTokenizer
+    - https://github.com/huggingface/tokenizers/blob/main/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
+    - 特征
+        - 算法：BERT的编码器是 BPE-WordPiece，将单词拆分成多个前缀符号（比如BERT中的##）最小单元
+        - 词典：有##开头的token，表示subword，
+            - 中文采用char粒度分词
+            - 英文采用  WordPiece
+
+
+
+
+    ## google/sentencepiece
+    - https://github.com/google/sentencepiece/blob/3863f7648e5d8edb571ac592f3ac4f5f0695275a/src/sentencepiece_model.proto#L48
+    - 支持 sentencepiece 和 wordpiece
+        - sentencepiece 有byte-bpe吗？
+            - UNIGRAM = 1;  // Unigram language model with dynamic algorithm
+            - BPE = 2;      // Byte Pair Encoding
+            - WORD = 3;     // Delimitered by whitespace.
+            - CHAR = 4;     // tokenizes into character sequence
+        - wordpiece
+    - 特征：
+        - 训练: spm_train --model_type unigram/bpe/char/word
+        - 特殊符号： Ġ
+        - 文件: *.sp_model  或 *.model  (可选文件 .vocab，) spm简称   (其他格式比如 tokenizer.json是给hf_tokenizer兼容用的)
+        - 实现:
+            - 依赖: protobuf
+            - 训练: `import sentencepiece as spm; spm.SentencePieceTrainer.train` 或 `spm_train`
+            - 加载: `import sentencepiece as spm; spm.SentencePieceProcessor().Load(vocab_file)`
+            - 方法: 是SentencePieceProcessor类型，sp_model.id_to_piece，有tokenizer.json tokenizer.model，
+            - 分词:
+                - pre_tokenizers.ByteLevel(add_prefix_space=True, use_regex=False)
+        - 词典:  词典字符有 ▁  (U+2581) ，表示空格或句首。
+    - 示例：google-t5, llama，baichuan, orion,
+        - llama: tokenizer.json(包含model.vocab model.merges)  tokenizer.model
+        - grok: 原始是 .model文件，后面转成了 tokenizer.json
+        - google-t5: tokenizer.json, spiece.model
+        - Skywork-13B-Math: tokenizer.model
+        - xlm_roberta: sentencepiece.bpe.model
+        - GPT2Tokenizer
+            - tokenizer.json, vocab.json, merges.txt   (https://huggingface.co/openai-community/gpt2)
+            - vocab.bpe, encoder.json, dict.txt  （fairseq版本，不常用，可以忽略这个版本）
+
+
+
+    ## thu/icetk
+      - icetk： sentencepiece的分支，支持image_tokenizer。
+    - glm, chatglm1, chatglm2
+
+    ## huggingface/tokenizers
+    - https://github.com/huggingface/tokenizers
+    - VS sentencepiece
+        - 支持sentencepiece
+            - .model转化为 (merges.txt + vocab.json) 或者 tokenizer.json
+                - https://github.com/huggingface/tokenizers/blob/main/bindings/python/scripts/sentencepiece_extractor.py
+            - 加载 merges.txt, vocab.json
+                - SentencePieceBPETokenizer  https://github.com/huggingface/tokenizers/blob/v0.19.1/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py#L10
+        - 在 sentencepiece基础上，hf_tokenizer支持pre-tokenization的正则表达式，对tab和换行支持更好，支持special token
+    - 类型： 支持 BBPE, WordPiece or Unigram
+    - 特征：
+        - 文件: tokenizer.json(包含后两个文件的内容), merges.txt, vocab.json
+            - added_tokens 在vocab中不一定存在。
+        - 实现:
+            - 训练: `from tokenizers.trainers import BpeTrainer, UnigramTrainer, WordLevelTrainer, WordPieceTrainer`
+            - 加载:
+            - 方法: .model.from_file  .model.save   .model.token_to_id  .model.tokenize
+        - .model 是 tokenizer.models.BPE 类型
+        - 词典有 Ġ  "\u0120" 开头
+        - 优势
+        -
+    - 示例：gpt2, gpt_neox_20b, moss, bloom, qwen2
+    - 优势：相对sentence piece，
+        - ss
+
+    ## openai/tiktoken
+    - 特征：空格就是空格，
+    - 示例：gpt3.5 gpt4, qwen,
+    """
+
+    """ 算法体系  https://www.huaxiaozhuan.com/%E5%B7%A5%E5%85%B7/huggingface_transformer/chapters/1_tokenizer.html
+    - word-base tokenizer:
+    - char-base tokenizer:
+    - subword-based Tokenizer
+        - BPE 
+            - byte-bpe: base vocabulary大小是256
+        - WordPiece:
+            - 相比BPE，WordPiece 仅保存最终词表，而不保存学到的 merge rule
+        - Unigram
+    - SentencePiece
+    
+    """
+
+    # 分类体系：https://github.com/huggingface/tokenizers/blob/main/bindings/python/py_src/tokenizers/implementations/
+    BertTokenizer = "wordpiece.BertTokenizer"
+    JapaneseTokenizer = (
+        "wordpiece.MecabTokenizer",
+        "https://github.com/polm/fugashi",
+    )  # 常用日语包 ipadic，fugashi，
+    ByteLevelBPETokenizer = "byte_level_bpe"  # BBPE
+    SentencePieceBPETokenizer = "sentencepiece_bpe"
+
+    # 分类体系
+
+    # SentencePeice(BPE)
+    SentencePiece = auto()  # sentencepiece.bpe, sentencepiece.unigram, sentencepiece.char, sentencepiece.word,
+    byte_level_bpe = auto()
+    # HFTokenizer = auto()  # , 支持
+    TikToken = auto()
+    # subword-nmt
+    # WordPiece
+
+
+# load_vocab_with_SPECIAL_TOKEN = True # 如果不包含会导致计算词典大小错误、overlap_token计算不一致。
+
+
+@dataclass
+class TokenizerConfig:
+    """
+    https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/leaderboard/read_evals.py
+    """
+
+    name_or_path: str  # org/model (path on hub), as unique id
+    name_display: str = None  #
+    impl: TokenizerImpl = None  # implementation, tokenizer_class/type
+    org: str = None
+    link: str = None  # http://**
+    desc: str = None  # description
+    meta: str = None
+    level: str = None  # char-level, word-level, byte-level
+    lang: str = None
+    init_kwargs: Dict[str, Any] = field(
+        default_factory=dict,
+    )
+
+    def __post_init__(self):
+        if self.link is None:
+            self.link = "https://huggingface.co/" + self.name_or_path  # TODO + revision
+        if self.name_display is None:
+            self.name_display = self.name_or_path
+
+    @classmethod
+    def init_from_json_file(cls, json_filepath: str) -> "TokenizerConfig":
+        pass
+
+    def __eq__(self, other):
+        if isinstance(other, self.__class__):
+            return self.__dict__ == other.__dict__
+        else:
+            return False
+
+    def __hash__(self):
+        return hash(self.name_or_path)
+
+
+# TODO: append link and description to the end of dropdown button.
+# Add tokenizer_class/type, comments
+_all_tokenizer_config = [
+    # bert style tokenizers
+    TokenizerConfig(
+        "google-bert/bert-base-cased",
+        impl=TokenizerImpl.BertTokenizer,
+        org="Google",
+        desc="first add whitespace around any CJK character, then perform wordpiece tokenization.",
+    ),
+    TokenizerConfig(
+        "google-bert/bert-base-uncased",
+        impl=TokenizerImpl.BertTokenizer,
+        org="Google",
+        desc="first add whitespace around any CJK character, then perform wordpiece tokenization.",
+    ),
+    TokenizerConfig(
+        "google-bert/bert-base-chinese",
+        impl=TokenizerImpl.BertTokenizer,
+        org="Google",
+        desc="first add whitespace around any CJK character, then perform wordpiece tokenization.",
+    ),
+    TokenizerConfig(
+        "google-bert/bert-base-german-cased",
+        impl=TokenizerImpl.BertTokenizer,
+        org="Google",
+    ),
+    TokenizerConfig(
+        "dbmdz/bert-base-german-uncased", impl=TokenizerImpl.BertTokenizer, org="dbmdz"
+    ),
+    TokenizerConfig(
+        "asafaya/bert-base-arabic", impl=TokenizerImpl.BertTokenizer, org="-"
+    ),
+    TokenizerConfig(
+        "google-bert/bert-base-multilingual-uncased",
+        impl=TokenizerImpl.BertTokenizer,
+        org="Google",
+    ),
+    TokenizerConfig(
+        "google-bert/bert-base-multilingual-cased",
+        impl=TokenizerImpl.BertTokenizer,
+        org="Google",
+    ),
+    TokenizerConfig(
+        "tohoku-nlp/bert-base-japanese",
+        impl=TokenizerImpl.BertTokenizer,
+        org="Tohoku",
+        desc="The texts are first tokenized by MeCab morphological parser with the IPA dictionary, "
+        "then split into subwords by the WordPiece algorithm.",
+    ),
+    TokenizerConfig(
+        "clue/roberta_chinese_clue_tiny",
+        name_display="clue/roberta-chinese-clue",
+        impl=TokenizerImpl.BertTokenizer,
+        org="CLUE",
+        init_kwargs={"revision": "refs/pr/1"},
+        desc="",
+        meta="去掉了繁体字, https://github.com/CLUEbenchmark/CLUEPretrainedModels/blob/master/README.md",
+    ),
+    TokenizerConfig(
+        "eson/kplug-base-encoder",
+        name_display="eson/kplug",
+        impl=TokenizerImpl.BertTokenizer,
+        org="JD",
+    ),
+    TokenizerConfig(
+        "ckiplab/gpt2-base-chinese", impl=TokenizerImpl.BertTokenizer, org="SINICA"
+    ),  # 台湾中央研究院
+    # WoBERT  https://kexue.fm/archives/7758
+    # WoBERT Plus  https://github.com/ZhuiyiTechnology/WoBERT
+    # gpt2 style tokenizers
+    TokenizerConfig(
+        "openai-community/gpt2", impl=TokenizerImpl.SentencePiece, org="OpenAI"
+    ),
+    # byte-level BPE,没有byte，是unicode-level的吗？
+    TokenizerConfig(
+        "ClassCat/gpt2-base-french", impl=TokenizerImpl.SentencePiece, org="ClassCat"
+    ),
+    TokenizerConfig(
+        "ClassCat/gpt2-base-spanish", impl=TokenizerImpl.SentencePiece, org="ClassCat"
+    ),
+    TokenizerConfig(
+        "fnlp/moss-moon-003-sft",
+        impl=TokenizerImpl.SentencePiece,
+        init_kwargs={"revision": "refs/pr/6"},
+        org="Fudan",
+        desc="This tokenizer has been trained to treat spaces like parts of the tokens "
+        "(a bit like sentencepiece) so a word will be encoded differently whether "
+        "it is at the beginning of the sentence (without space) or not",
+        meta="在gpt2词典基础上，扩充了5万中文",
+    ),
+    TokenizerConfig(
+        "bigscience/bloom",
+        impl=TokenizerImpl.SentencePiece,
+        org="BigScience",
+        meta="比gpt_neox的词典 对中文支持更好。",
+    ),
+    # ("bloomz_6b4_zh",
+    # ("BelleGroup/BELLE-7B-2M",   # 模型和词典都基于bloom
+    #
+    TokenizerConfig(
+        "EleutherAI/gpt-neox-20b", impl=TokenizerImpl.SentencePiece, org="EleutherAI"
+    ),  # 5万
+    TokenizerConfig(
+        "cyberagent/open-calm-7b", impl=TokenizerImpl.SentencePiece, org="CyberAgent"
+    ),  # GPTNeoXTokenizer
+    TokenizerConfig(
+        "abeja/gpt-neox-japanese-2.7b", impl=TokenizerImpl.SentencePiece, org="ABEJA"
+    ),
+    TokenizerConfig(
+        "rinna/bilingual-gpt-neox-4b",
+        impl=TokenizerImpl.SentencePiece,
+        org="ABEJA",
+        lang="en/ja",
+    ),
+    TokenizerConfig(
+        "Qwen/Qwen1.5-14B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
+    ),  # 15万，速度有点慢
+    TokenizerConfig(
+        "Qwen/Qwen1.5-110B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
+    ),
+    TokenizerConfig(
+        "Qwen/Qwen1.5-1.8B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
+    ),
+    TokenizerConfig("Qwen/Qwen2-0.5B", impl=TokenizerImpl.SentencePiece, org="Alibaba"),
+    TokenizerConfig("Qwen/Qwen2-72B", impl=TokenizerImpl.SentencePiece, org="Alibaba"),
+    TokenizerConfig(
+        "Qwen/Qwen2.5-0.5B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
+    ),
+    TokenizerConfig(
+        "Qwen/Qwen2.5-72B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
+    ),
+    TokenizerConfig(
+        "HuggingFaceH4/starchat-alpha", impl=TokenizerImpl.SentencePiece, org="-"
+    ),
+    ####### google/sentencepiece tokenizer:
+    # T5 llama internlm
+    TokenizerConfig(
+        "google-t5/t5-large",
+        name_display="google-t5/t5",
+        impl=TokenizerImpl.SentencePiece,
+        org="Google",
+    ),
+    # t5_small, t5_base, t5_large, flan_t5_base,
+    # ("t5_base", "", "sentencepiece"),
+    # TokenizerConfig("google/flan-t5-base", impl=TokenizerImpl.SentencePiece, ),
+    TokenizerConfig(
+        "lmsys/fastchat-t5-3b-v1.0",
+        impl=TokenizerImpl.SentencePiece,
+        org="LMSYS",
+        init_kwargs={
+            "use_fast": False
+        },  # 解决 pyo3_runtime.PanicException: AddedVocabulary bad split
+    ),
+    TokenizerConfig(
+        "CohereForAI/aya-101", org="Cohere For AI"
+    ),  # "tokenizer_class": "T5Tokenizer",
+    TokenizerConfig(
+        "ClueAI/ChatYuan-large-v2", impl=TokenizerImpl.SentencePiece, org="CLUE"
+    ),
+    TokenizerConfig(
+        "ClueAI/PromptCLUE-base", impl=TokenizerImpl.SentencePiece, org="CLUE"
+    ),
+    # byte-level BPE
+    # '中文单字': 700, '中文多字': 0  meta-llama/Meta-Llama-3.1-405B
+    #
+    TokenizerConfig(
+        "meta-llama/Llama-3.2-1B-Instruct", impl=TokenizerImpl.SentencePiece, org="Meta"
+    ),
+    TokenizerConfig(
+        "meta-llama/Llama-3.2-3B-Instruct", impl=TokenizerImpl.SentencePiece, org="Meta"
+    ),
+    # TokenizerConfig("meta-llama/Llama-3.3-70B-Instruct", impl=TokenizerImpl.SentencePiece,
+    #                 org="Meta"),
+    TokenizerConfig(
+        "meta-llama/Meta-Llama-3.1-405B", impl=TokenizerImpl.SentencePiece, org="Meta"
+    ),
+    TokenizerConfig(
+        "NousResearch/Hermes-3-Llama-3.1-405B",
+        impl=TokenizerImpl.SentencePiece,
+        org="NousResearch",
+    ),
+    TokenizerConfig(
+        "gradientai/Llama-3-8B-Instruct-Gradient-1048k",
+        name_display="Meta/llama3",
+        impl=TokenizerImpl.SentencePiece,
+        org="Meta",
+        desc="llama split all numbers into individual digits, and fallback to bytes to decompose unknown UTF-8 characters",
+    ),
+    TokenizerConfig(
+        "NousResearch/Llama-2-7b-chat-hf",
+        name_display="Meta/llama2",
+        impl=TokenizerImpl.SentencePiece,
+        org="Meta",
+    ),
+    TokenizerConfig(
+        "huggyllama/llama-7b",
+        name_display="Meta/llama",
+        impl=TokenizerImpl.SentencePiece,
+        org="Meta",
+    ),
+    TokenizerConfig(
+        "hpcai-tech/grok-1",
+        name_display="xai-org/grok-1",
+        impl=TokenizerImpl.SentencePiece,
+        org="xAI",
+    ),
+    # 由.model文件转化为了
+    TokenizerConfig(
+        "hfl/chinese-llama-lora-7b",
+        impl=TokenizerImpl.SentencePiece,
+        org="-",
+        meta="向原始LLaMA的词汇表中添加2w个中文词汇，针对原版LLaMA模型扩充了中文词表， 提升了中文编解码效率",
+    ),
+    #
+    TokenizerConfig(
+        "hfl/chinese-llama-2-7b",
+        impl=TokenizerImpl.SentencePiece,
+        org="-",
+        meta="重新设计了新词表（大小：55296），进一步提升了中文字词的覆盖程度",
+    ),  #
+    TokenizerConfig(
+        "hfl/llama-3-chinese-8b", impl=TokenizerImpl.SentencePiece, org="-"
+    ),
+    TokenizerConfig(
+        "hfl/chinese-alpaca-lora-7b", impl=TokenizerImpl.SentencePiece, org="-"
+    ),
+    # 中文Alpaca模型在上述中文LLaMA模型的基础上进一步使用了指令数据进行精调。  "比chinese_llama词典多一个`[PAD]`，请勿混用"
+    #
+    # ("belle_llama_ext_7b",
+    # ("alpaca_7b",
+    TokenizerConfig(
+        "baichuan-inc/Baichuan-7B",
+        name_display="baichuan-inc/baichuan",
+        impl=TokenizerImpl.SentencePiece,
+        level="byte-level",
+        org="Baichuan",
+    ),
+    TokenizerConfig(
+        "baichuan-inc/Baichuan2-7B-Chat",
+        name_display="baichuan-inc/baichuan2",
+        impl=TokenizerImpl.SentencePiece,
+        org="Baichuan",
+        desc="expand the vocabulary size from 64000 in Baichuan1 to 125696",
+    ),
+    TokenizerConfig(
+        "internlm/internlm-chat-7b",
+        impl=TokenizerImpl.SentencePiece,
+        org="Shanghai AI Lab",
+    ),
+    # 上海AI实验室 +  商汤
+    TokenizerConfig(
+        "internlm/internlm2-chat-7b",
+        impl=TokenizerImpl.SentencePiece,
+        org="Shanghai AI Lab",
+    ),
+    TokenizerConfig(
+        "internlm/internlm2-math-7b",
+        impl=TokenizerImpl.SentencePiece,
+        org="Shanghai AI Lab",
+    ),
+    TokenizerConfig(
+        "internlm/internlm-xcomposer-7b",
+        impl=TokenizerImpl.SentencePiece,
+        org="Shanghai AI Lab",
+    ),
+    TokenizerConfig("tiiuae/falcon-7b", impl=TokenizerImpl.SentencePiece, org="TII"),
+    TokenizerConfig("tiiuae/falcon-180b", impl=TokenizerImpl.SentencePiece, org="TII"),
+    TokenizerConfig(
+        "Skywork/Skywork-13B-base", impl=TokenizerImpl.SentencePiece, org="Kunlun"
+    ),
+    TokenizerConfig(
+        "Skywork/Skywork-13B-Math", impl=TokenizerImpl.SentencePiece, org="Kunlun"
+    ),  # 文件：tokenizer.model
+    TokenizerConfig(
+        "FacebookAI/xlm-roberta-base", impl=TokenizerImpl.SentencePiece, org="Facebook"
+    ),
+    # 这个的tokenizer.json 为什么没有merges? vocab里为什么有概率值？
+    # "goat",
+    # ##### glm系列
+    # "glm_chinese",),
+    TokenizerConfig(
+        "THUDM/chatglm-6b",
+        impl=TokenizerImpl.SentencePiece,
+        org="Tsinghua",
+        meta=f"num_image_tokens: {12}; num_image_tokens: {34} ",
+        init_kwargs={"revision": "refs/pr/100"},
+    ),
+    TokenizerConfig(
+        "THUDM/chatglm2-6b",
+        impl=TokenizerImpl.SentencePiece,
+        org="Tsinghua",
+    ),
+    TokenizerConfig(
+        "THUDM/chatglm3-6b",
+        impl=TokenizerImpl.SentencePiece,
+        org="Tsinghua",
+    ),
+    TokenizerConfig(
+        "thu-coai/CharacterGLM-6B",
+        impl=TokenizerImpl.SentencePiece,
+        org="Tsinghua",
+    ),
+    # tiktoken 系列
+    TokenizerConfig(
+        "openai/text-davinci-003",
+        impl=TokenizerImpl.TikToken,
+        org="OpenAI",
+        link="https://github.com/openai/tiktoken",
+    ),
+    #
+    TokenizerConfig(
+        "openai/code-davinci-002",
+        impl=TokenizerImpl.TikToken,
+        org="OpenAI",
+        link="https://github.com/openai/tiktoken",
+    ),
+    TokenizerConfig(
+        "openai/gpt-3.5-turbo",
+        impl=TokenizerImpl.TikToken,
+        org="OpenAI",
+        link="https://github.com/openai/tiktoken",
+        desc="tiktoken is a fast BPE tokeniser for use with OpenAI's models. There are 16 tokens KeyError",
+    ),
+    TokenizerConfig(
+        "openai/gpt-4",
+        impl=TokenizerImpl.TikToken,
+        org="OpenAI",
+        link="https://github.com/openai/tiktoken",
+    ),
+    TokenizerConfig(
+        "openai/gpt-4o",
+        impl=TokenizerImpl.TikToken,
+        org="OpenAI",
+        link="https://github.com/openai/tiktoken",
+    ),
+    TokenizerConfig(
+        "Qwen/Qwen-7B-Chat",
+        name_display="Qwen/Qwen",
+        impl=TokenizerImpl.TikToken,
+        org="Alibaba",
+        init_kwargs={"revision": "refs/pr/56"},
+        meta="在gpt4词典基础上，删除了100个多数字token，增加10000中文词token；并优化了special_token的分词",
+    ),
+    # https://huggingface.co/Qwen/Qwen-7B-Chat#%E6%A8%A1%E5%9E%8B%E7%BB%86%E8%8A%82%EF%BC%88model%EF%BC%89
+    #  该词表在GPT-4使用的BPE词表cl100k_base基础上，对中文、多语言进行了优化，在对中、英、代码数据的高效编解码的基础上，
+    #  对部分多语言更加友好，方便用户在不扩展词表的情况下对部分语种进行能力增强。 词表对数字按单个数字位切分。
+    # TokenizerConfig("Qwen/Qwen-72B-Chat", impl=TokenizerImpl.TikToken),
+    # 未分类
+    # ("amber", ""),
+    TokenizerConfig("LLM360/CrystalCoder", org="MBZUAI"),
+    TokenizerConfig("apple/DCLM-7B", org="Apple"),
+    TokenizerConfig("mistralai/Mistral-7B-v0.1", org="Mistral"),
+    TokenizerConfig("mistralai/Mixtral-8x7B-v0.1", org="Mistral"),
+    TokenizerConfig("mistralai/Mistral-Large-Instruct-2407", org="Mistral"),
+    TokenizerConfig("mistralai/Mistral-Nemo-Instruct-2407", org="Mistral"),
+    TokenizerConfig("paust/pko-t5-large", org="PAUST"),
+    TokenizerConfig("01-ai/Yi-6B", org="Yi"),
+    TokenizerConfig("01-ai/Yi-34B", org="Yi"),
+    TokenizerConfig("01-ai/Yi-VL-34B", org="Yi"),
+    TokenizerConfig("01-ai/Yi-1.5-34B", org="Yi"),
+    TokenizerConfig("OrionStarAI/Orion-14B-Chat", org="OrionStar"),
+    TokenizerConfig("microsoft/phi-1", org="Microsoft"),
+    TokenizerConfig("microsoft/phi-2", org="Microsoft"),
+    TokenizerConfig(
+        "microsoft/Phi-3-mini-4k-instruct", org="Microsoft", meta="即llama vocab"
+    ),
+    TokenizerConfig("Upstage/SOLAR-10.7B-v1.0", org="-"),
+    TokenizerConfig("google/mobilebert-uncased", org="Google"),
+    # ("google/mobilenet_v2_1.0_224",),  # error
+    TokenizerConfig("google/switch-c-2048", org="Google"),
+    TokenizerConfig("google/byt5-small", org="Google"),
+    TokenizerConfig("google/mt5-large", org="Google"),
+    TokenizerConfig("WizardLM/WizardCoder-Python-7B-V1.0", org="Microsoft"),
+    TokenizerConfig("WizardLM/WizardCoder-15B-V1.0", org="Microsoft"),
+    TokenizerConfig("WizardLM/WizardLM-7B-V1.0", org="Microsoft"),
+    TokenizerConfig("WizardLM/WizardMath-70B-V1.0", org="Microsoft"),
+    TokenizerConfig("TigerResearch/tigerbot-70b-chat-v4-4k", org="Tigerobo"),
+    TokenizerConfig("TigerResearch/tigerbot-13b-chat-v2", org="Tigerobo"),
+    TokenizerConfig("deepseek-ai/deepseek-coder-33b-instruct", org="DeepSeek"),
+    TokenizerConfig("deepseek-ai/deepseek-llm-7b-base", org="DeepSeek"),
+    TokenizerConfig("deepseek-ai/DeepSeek-V2", org="DeepSeek"),
+    TokenizerConfig("deepseek-ai/DeepSeek-V3", org="DeepSeek"),
+    TokenizerConfig(
+        "deepseek-ai/DeepSeek-R1", org="DeepSeek"
+    ),  # 在llama3的词典上，增加了一些中文token，删掉了一部分token
+    TokenizerConfig("deepseek-ai/DeepSeek-R1-Zero", org="DeepSeek"),
+    TokenizerConfig("deepseek-ai/DeepSeek-R1-Distill-Llama-70B", org="DeepSeek"),
+    TokenizerConfig("google/gemma-7b", org="Google"),
+    TokenizerConfig("google/gemma-2-9b", org="Google"),
+    TokenizerConfig("allenai/OLMo-7B-hf", org="Allen AI"),
+    TokenizerConfig("HuggingFaceH4/zephyr-7b-beta", org="HuggingFace"),
+    TokenizerConfig("ai21labs/Jamba-v0.1", org="AI21"),
+    TokenizerConfig("databricks/dbrx-instruct", org="Databricks"),
+    TokenizerConfig("MiniMaxAI/MiniMax-Text-01", org="MiniMax"),
+    # TokenizerConfig("nvidia/Nemotron-4-340B-Instruct", org="Nvidia"),
+    # ("claude",),
+    # https://github.com/Duxiaoman-DI/XuanYuan
+    # https://huggingface.co/apple/OpenELM-3B-Instruct  https://huggingface.co/apple/OpenELM-3B
+]
+
+assert len(set([config.name_display for config in _all_tokenizer_config])) == len(
+    _all_tokenizer_config
+)
+assert len(set([config.name_or_path for config in _all_tokenizer_config])) == len(
+    _all_tokenizer_config
+)
+assert len(
+    set([config.name_or_path.split("/")[-1] for config in _all_tokenizer_config])
+) == len(_all_tokenizer_config)
+
+
+class TokenizerFactory:
+    def __init__(self):
+        # self.all_tokenizer_configs = sorted(_all_tokenizer_config, key=lambda k: k.name_or_path)
+        self.all_tokenizer_configs = sorted(
+            _all_tokenizer_config, key=lambda k: k.name_display
+        )
+        self.all_tokenizer_names = [
+            config.name_or_path for config in self.all_tokenizer_configs
+        ]
+        self.name_to_config_list = [
+            {config.name_or_path: config for config in self.all_tokenizer_configs},
+            {config.name_display: config for config in self.all_tokenizer_configs},
+            {
+                config.name_display.split("/")[-1]: config
+                for config in self.all_tokenizer_configs
+            },
+        ]
+        self.tokenizer_cache = {}
+
+    def get_tokenizer_config(self, tokenizer_name: str) -> TokenizerConfig:
+        for name_to_config in self.name_to_config_list:
+            if tokenizer_name in name_to_config:
+                return name_to_config[tokenizer_name]
+        return None
+
+    def get_tokenizer(self, tokenizer_name: str):
+        """
+        :param tokenizer_name:
+        :return:
+        """
+        tokenizer_config = self.get_tokenizer_config(tokenizer_name)
+
+        # 1. load from cache
+        if tokenizer_config in self.tokenizer_cache:
+            return self.tokenizer_cache[tokenizer_config]
+
+        # 2. load tokenizer
+        tokenizer = self.load_tokenizer(tokenizer_config)
+
+        self.tokenizer_cache[tokenizer_config] = tokenizer
+        return tokenizer
+
+    def get_name_with_hyperlink(self, tokenizer_name: str) -> str:
+        def model_hyperlink(link, model_name):
+            model_name = model_name
+            return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+        tokenizer_config = self.get_tokenizer_config(tokenizer_name)
+        return model_hyperlink(
+            tokenizer_config.link, tokenizer_config.name_display.split("/")[-1]
+        )
+
+    def load_tokenizer(self, tokenizer_config):
+        if tokenizer_config == None:
+            print("dd")
+        logger.info(f"loading tokenizer {tokenizer_config.name_or_path}")
+        if (
+            tokenizer_config.impl == TokenizerImpl.TikToken
+            and "openai" in tokenizer_config.name_or_path
+        ):
+            tokenizer = tiktoken.encoding_for_model(
+                tokenizer_config.name_or_path.replace("openai/", "")
+            )
+        else:
+            tokenizer = AutoTokenizer.from_pretrained(
+                tokenizer_config.name_or_path,
+                trust_remote_code=True,
+                **tokenizer_config.init_kwargs,
+            )
+        return tokenizer
+
+    def add_config(
+        self,
+    ):
+        pass
+
+    def add_tokenizer(self, tokenizer_name):
+        pass
+
+
+tokenizer_factory = TokenizerFactory()
+
+
+def add_tokenizer(tokenizer_name: str):
+    """
+    :param tokenizer_name:
+    :return:
+    """
+    if tokenizer_name in []:
+        logger.info(f"{tokenizer_name} already exits")
+    else:
+        # add to config
+        tokenizer_config = TokenizerConfig(tokenizer_name, org="-")
+
+        # add to tokenizer
+        tokenizer = tokenizer_factory.load_tokenizer(tokenizer_config)
+
+        # refresh cache
+
+        try:
+            tokenizer = AutoTokenizer.from_pretrained(
+                tokenizer_name, trust_remote_code=True, **tokenizer_config.init_kwargs
+            )
+            tokenizer_factory.all_tokenizer_configs.append(
+                "",
+            )
+            tokenizer_factory
+
+        except Exception as e:
+            logger.error(e)
+
+    pass
+
+
+# class TokenizerType(Enum):
+#
+#     # BERTTokenizer
+#     # 依赖一个txt文件
+#
+#
+#     # https://github.com/EleutherAI/gpt-neox/blob/v2.0/megatron/tokenizer/tokenizer.py#L231
+#     # 依赖一个json文件，Tokenizer.from_file(vocab_file)
+#     # 案例：gpt-neox-20B
+#     HFTokenizer = auto()
+#
+#     # 依赖: model_file, sentencepiece.SentencePieceProcessor(model_file)
+#     # 案例：
+#     SentencePieceTokenizer = auto()
+#
+#
+#     # 依赖: 3个json文件：vocab.json, merges.txt, special_tokens.txt
+#     # 源码:
+#     #   - https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/tokenizer/gpt2_tokenization.py#L92
+#     # Byte-level BPE
+#     GPT2BPETokenizer = auto()
+
+
+if __name__ == "__main__":
+    for tokenizer_config in tokenizer_factory.all_tokenizer_configs:
+        if True:
+            # if "t5" in tokenizer_config.name_or_path:
+            tokenizer1 = tokenizer_factory.get_tokenizer(tokenizer_config.name_or_path)
+            tokenizer2 = tokenizer_factory.get_tokenizer(tokenizer_config.name_display)
+            tokenizer3 = tokenizer_factory.get_tokenizer(
+                tokenizer_config.name_display.split("/")[-1]
+            )
+            assert tokenizer1 == tokenizer2 == tokenizer3
+            print(tokenizer_config.name_or_path, len(tokenizer1))