chore: cleanup

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2025 Athens NLP Summer School
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

README.md

@@ -1,12 +0,0 @@
----
-title: Tokenization Playground
-emoji: 📝
-colorFrom: indigo
-colorTo: purple
-sdk: gradio
-pinned: false
-short_description: Compare different tokenizers
----
-
-# tokenization_playground
-Link to source code: https://github.com/athnlp/tokenization_playground

app.py

@@ -1,25 +0,0 @@
-import os
-
-import gradio as gr
-from huggingface_hub import login
-
-from playground_app import demo as playground_tab
-
-auth_token = os.environ.get("HF_TOKEN", None)
-if auth_token:
-    login(token=auth_token)
-
-
-title = """
-<div align="center">
-    <span>Tokenization Playground</span>
-</div>
-"""
-
-with gr.Blocks() as demo:
-    _ = gr.HTML(f"<h1 style='text-align: center; margin-bottom: 1rem'>{title}</h1>")
-    _ = playground_tab.render()
-
-if __name__ == "__main__":
-    demo.launch()
-    # demo.launch(share=True)

character_util.py

@@ -1,178 +0,0 @@
-import json
-import os
-from pathlib import Path
-from typing import Literal
-
-import numpy as np
-import pandas as pd
-from utils.lang_util import detect_language_by_unicode, language_ranges
-from utils.log_util import logger
-from utils.text_util import contains_digit, get_space_count
-from vocab import tokenizer_factory
-
-CURRENT_DIR = Path.parent(Path.resolve(__file__))
-
-cache = {}
-default_columns = ["digit", "zh"]
-
-
-def text_to_unicode(text: str) -> str:
-    """Convert text to unicode representation."""
-    return "".join(rf"\u{ord(character):04X}" for character in text)
-
-
-def calculate_dist(token_lens: list[int]) -> str:
-    """Calculate the distribution of token lengths."""
-    if not token_lens:
-        return "-"
-    return f"{min(token_lens)},{round(np.median(token_lens))},{max(token_lens)}"
-
-
-def iter_vocab(
-    tokenizer_name: str,
-    from_cache: bool = True,
-    cache_dir: str = "stats",
-) -> pd.DataFrame | dict:
-    """:param tokenizer_name:
-    :param from_cache:
-    :param cache_dir:
-    :return:
-    """
-    tokenizer_config = tokenizer_factory.get_tokenizer_config(tokenizer_name)
-
-    cache_dir = os.path.join(CURRENT_DIR, cache_dir)
-    os.makedirs(cache_dir, exist_ok=True)
-
-    # load from cache
-    cache_path = os.path.join(cache_dir, "character_stats.json")
-    if not cache and os.path.exists(cache_path):
-        with open(cache_path, encoding="utf-8") as f_tmp:
-            cache.update(json.load(f_tmp))
-    if from_cache and tokenizer_name in cache:
-        # logger.info(f"load {tokenizer_config.name_or_path} from cache")
-        return cache[tokenizer_name]
-
-    tokenizer = tokenizer_factory.get_tokenizer(tokenizer_name)
-
-    tokens_by_lang = {lang[1]: [] for lang in language_ranges}
-    digit_tokens = []
-    space_tokens = []
-    byte_tokens = []
-
-    buffer = []
-    for token_id in range(tokenizer.vocab_size):
-        # for token_id in tokenizer.get_vocab():
-        # for token_id in range(len(tokenizer)):
-        decode_str = tokenizer.decode([token_id], skip_special_tokens=False)
-        token = tokenizer.convert_ids_to_tokens([token_id], skip_special_tokens=False)[0]
-        tags = []
-        if token is None:  # 有些词典有空的id（不连续）
-            continue
-        if isinstance(token, bytes):
-            token = token.decode("utf-8", errors="ignore")
-
-        if hasattr(tokenizer, "sp_model") and tokenizer.sp_model.is_byte(token_id):
-            tags.append("is_byte")
-            byte_tokens.append(token)
-
-        language_tags = detect_language_by_unicode(decode_str)
-        for language in language_tags:
-            tokens_by_lang[language[1]].append(decode_str)
-
-        if contains_digit(decode_str):
-            tags.append("digit")
-            digit_tokens.append(decode_str)
-
-        space_count = get_space_count(decode_str)
-        if space_count > 0:
-            space_tokens.append(decode_str)
-
-        buffer.append(
-            json.dumps(
-                {
-                    "id": token_id,
-                    "token": token,
-                    "token_decode": decode_str,
-                    "token_dumps": json.dumps(token),
-                    "token_unicode": text_to_unicode(token),
-                    "token_len": len(decode_str),
-                },
-                ensure_ascii=False,
-            )
-            + "\n"
-        )
-
-    result = {
-        "tokenizer": tokenizer_factory.get_name_with_hyperlink(tokenizer_name),
-        "organization": tokenizer_config.org,
-        "vocab_size": len(tokenizer),
-        "num(digit)": len(digit_tokens),
-        "len(digit)": calculate_dist([len(token) for token in digit_tokens]),
-        "num(space)": len(space_tokens),
-        "len(space)": calculate_dist([len(token) for token in space_tokens]),
-    }
-
-    for lang, tokens in tokens_by_lang.items():
-        result[f"num({lang})"] = len(tokens)
-        result["len(" + lang + ")"] = calculate_dist([len(token) for token in tokens])
-
-    out_path = os.path.join(
-        cache_dir, f"iter_vocab/{tokenizer_name.replace('/', '_')}.vocab.jsonl"
-    )
-    with open(out_path, "w", encoding="utf-8") as f_out:
-        for line in buffer:
-            f_out.write(line)
-    len_before = len(cache)
-    cache[tokenizer_name] = result
-    len_after = len(cache)
-    logger.info(f"saving {tokenizer_name} to memory and file cache: {len_before}->{len_after}")
-    with open(cache_path, "w", encoding="utf-8") as f_out:
-        f_out.write(json.dumps(cache, ensure_ascii=False, indent=2))
-    return result
-
-
-def to_dataframe(stats: dict[str, Any], columns: list[str]) -> pd.DataFrame:
-    table = []
-    for stat in stats.values():
-        filtered_stat = {}
-        for k, v in stat.items():
-            if not k.startswith("num") and not k.startswith("len"):
-                filtered_stat[k] = v
-            if any(column in k for column in columns):
-                k = k.replace("ja-kana", "kana")
-                filtered_stat[k] = v
-        table.append(filtered_stat)
-    return pd.DataFrame(table)
-
-
-def get_character_table(
-    tokenizer_filter: str | None = None,
-    columns: list | None = None,
-    return_type: Literal["dict", "dataframe"] | None = "dataframe",
-) -> pd.DataFrame | dict:
-    logger.info(f"columns: {columns}, tokenizer_filter: {tokenizer_filter}")
-    stats = {}
-    if columns is None:
-        columns = default_columns
-    if tokenizer_filter is not None:
-        tokenizer_names = [
-            tokenizer_config.name_or_path
-            for tokenizer_config in tokenizer_factory.all_tokenizer_configs
-            if tokenizer_filter.lower() in tokenizer_config.name_or_path.lower()
-        ]
-    else:
-        tokenizer_names = tokenizer_factory.all_tokenizer_names
-
-    for tokenizer_name in tokenizer_names:
-        stat = iter_vocab(tokenizer_name)
-        stats[tokenizer_name] = stat
-
-    if return_type == "dataframe":
-        stats = to_dataframe(stats, columns)
-    return stats
-
-
-if __name__ == "__main__":
-    # aa = get_character_table(tokenizer_filter="baichuan")
-    df = get_character_table()
-    logger.info(f"\n{df.to_markdown(index=False)}")

playground_app.py

@@ -1,91 +0,0 @@
-import gradio as gr
-from playground_examples import examples
-from playground_util import on_load, tokenize, tokenize_pair
-from vocab import tokenizer_factory
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = JSON.stringify(Object.fromEntries(params));
-        return url_params;
-        }
-    """
-
-all_tokenizer_name = [
-    (config.name_display, config.name_or_path)
-    for config in tokenizer_factory.all_tokenizer_configs
-]
-
-with gr.Blocks() as demo:
-    with gr.Row():
-        gr.Markdown("## Input Text")
-        dropdown_examples = gr.Dropdown(
-            sorted(examples.keys()),
-            value="Examples",
-            type="index",
-            allow_custom_value=True,
-            show_label=False,
-            container=False,
-            scale=0,
-            elem_classes="example-style",
-        )
-    user_input = gr.Textbox(
-        label="Input Text",
-        lines=5,
-        show_label=False,
-    )
-
-    with gr.Row():
-        with gr.Column(scale=6), gr.Group():
-            tokenizer_name_1 = gr.Dropdown(all_tokenizer_name, label="Tokenizer 1")
-
-        with gr.Column(scale=6), gr.Group():
-            tokenizer_name_2 = gr.Dropdown(all_tokenizer_name, label="Tokenizer 2")
-
-    with gr.Row():
-        # dynamic change label
-        with gr.Column():
-            output_text_1 = gr.Highlightedtext(show_legend=False, show_inline_category=False)
-        with gr.Column():
-            output_text_2 = gr.Highlightedtext(show_legend=False, show_inline_category=False)
-
-    with gr.Row():
-        output_table_1 = gr.Dataframe()
-        output_table_2 = gr.Dataframe()
-
-    tokenizer_name_1.change(
-        tokenize, [user_input, tokenizer_name_1], [output_text_1, output_table_1]
-    )
-
-    tokenizer_name_2.change(
-        tokenize, [user_input, tokenizer_name_2], [output_text_2, output_table_2]
-    )
-
-    user_input.change(
-        tokenize_pair,
-        [user_input, tokenizer_name_1, tokenizer_name_2],
-        [output_text_1, output_table_1, output_text_2, output_table_2],
-        show_api=False,
-    )
-
-    dropdown_examples.change(
-        lambda example_idx: (
-            examples[sorted(examples.keys())[example_idx]]["text"],
-            examples[sorted(examples.keys())[example_idx]]["tokenizer_1"],
-            examples[sorted(examples.keys())[example_idx]]["tokenizer_2"],
-        ),
-        dropdown_examples,
-        [user_input, tokenizer_name_1, tokenizer_name_2],
-        show_api=False,
-    )
-
-    demo.load(
-        fn=on_load,
-        inputs=[user_input],
-        outputs=[user_input, tokenizer_name_1, tokenizer_name_2],
-        js=get_window_url_params,
-        show_api=False,
-    )
-
-if __name__ == "__main__":
-    demo.launch(share=True)

playground_examples.py

@@ -1,42 +0,0 @@
-default_user_input = """Replace this text in the input field to see how tokenization works."""
-default_tokenizer_name_1 = "openai/gpt-4o"
-default_tokenizer_name_2 = "Qwen/Qwen2.5-72B"
-
-
-number_example = """127+677=804
-127 + 677 = 804\n
-1275+6773 = 8041
-1275 + 6773 = 8048"""
-
-code_example = """for i in range(1, 101):
-    if i % 3 == 0 and i % 5 == 0:
-        print("FizzBuzz")
-    elif i % 3 == 0:
-        print("Fizz")
-    elif i % 5 == 0:
-        print("Buzz")
-    else:
-        print(i)
-"""
-
-spelling_example = """How do you spell "accommodate"?
-How many letters are in the word "accommodate"?
-How many r's are in the word strawberry?"""
-
-examples = {
-    "number": {
-        "text": number_example,
-        "tokenizer_1": default_tokenizer_name_1,
-        "tokenizer_2": default_tokenizer_name_2,
-    },
-    "code": {
-        "text": code_example,
-        "tokenizer_1": default_tokenizer_name_1,
-        "tokenizer_2": default_tokenizer_name_2,
-    },
-    "spelling": {
-        "text": spelling_example,
-        "tokenizer_1": default_tokenizer_name_1,
-        "tokenizer_2": default_tokenizer_name_2,
-    },
-}

playground_util.py

@@ -1,107 +0,0 @@
-import json
-from functools import lru_cache
-from typing import Any
-
-import gradio as gr
-import pandas as pd
-from playground_examples import (
-    default_tokenizer_name_1,
-    default_tokenizer_name_2,
-    default_user_input,
-)
-from utils.i18n_util import get_lang
-from utils.log_util import logger
-from vocab import tokenizer_factory
-
-
-@lru_cache
-def _tokenize(text: str, tokenizer_name: str, color_num: int = 5, add_special_token: bool = False):
-    logger.info(
-        "param=" + json.dumps({"text": text, "tokenizer_type": tokenizer_name}, ensure_ascii=False)
-    )
-    pos_tokens = []
-    tokenizer = tokenizer_factory.get_tokenizer(tokenizer_name)
-    encoding = tokenizer.encode(text) if add_special_token else tokenizer.encode(text)
-    table = []
-
-    for idx, token_id in enumerate(encoding):
-        decoded_text = tokenizer.decode([token_id])
-        decoded_text = decoded_text.replace(
-            " ", "⋅"
-        )  # replace space with ⋅ for better visualization
-        pos_tokens.extend([(decoded_text, str(idx % color_num))])
-
-        try:
-            token = tokenizer.decode([token_id])[0]
-        except:
-            token = {v: k for k, v in tokenizer.get_vocab().items()}[token_id]
-
-        if isinstance(token, bytes):
-            try:
-                token_str = token.decode("utf-8")
-            except:
-                token_str = token.decode("utf-8", errors="ignore")
-                logger.error(
-                    f"{idx}: decode_error: "
-                    + json.dumps(  # gpt_35_turbo 经常有token会decode error，这里用来记录一下
-                        {
-                            "tokenizer_type": tokenizer_name,
-                            "token": str(token),
-                            "token_str": token_str,
-                        },
-                        ensure_ascii=False,
-                    )
-                )
-
-            # json_dumps = json.dumps(token_str)
-        elif isinstance(token, str):
-            token_str = token
-        else:
-            logger.error(
-                f"{idx}: wrong type for token {token_id} {type(token)} "
-                + json.dumps({"text": text, "tokenizer_type": tokenizer_name}, ensure_ascii=False)
-            )
-            token_str = token
-
-        table.append({"TokenID": token_id, "Text": decoded_text})
-
-    table_df = pd.DataFrame(table)
-    logger.info(f"tokenizer_type={tokenizer_name}, Tokens={table[:4]}")
-    return pos_tokens, len(encoding), table_df
-
-
-def tokenize(
-    text: str, tokenizer_name: str, color_num: int = 5
-) -> tuple[dict[Any, Any], pd.DataFrame]:
-    """Tokenize an input text."""
-    pos_tokens, num_tokens, table_df = _tokenize(text, tokenizer_name, color_num)
-    return gr.update(value=pos_tokens, label=f"Tokens: {num_tokens}"), table_df
-
-
-def tokenize_pair(text, tokenizer_type_1, tokenizer_type_2, color_num: int = 5):
-    """input_text.change."""
-    pos_tokens_1, table_df_1 = tokenize(text, tokenizer_type_1, color_num)
-    pos_tokens_2, table_df_2 = tokenize(text, tokenizer_type_2, color_num)
-    return pos_tokens_1, table_df_1, pos_tokens_2, table_df_2
-
-
-def on_load(url_params: str, request: gr.Request = None) -> tuple[str, str, str]:
-    """Function triggered on page load to get URL parameters."""
-    text = default_user_input
-    tokenizer_type_1 = default_tokenizer_name_1
-    tokenizer_type_2 = default_tokenizer_name_2
-    try:
-        url_params_dict = json.loads(url_params)
-    except json.JSONDecodeError:
-        url_params_dict = {}
-
-    if request:
-        lang, _ = get_lang(request)
-        logger.info(str(request.headers))
-        client_ip = request.client.host
-
-        tokenizer_type_1 = url_params_dict.get("tokenizer1", default_tokenizer_name_1)
-        tokenizer_type_2 = url_params_dict.get("tokenizer2", default_tokenizer_name_2)
-        text = url_params_dict.get("text", default_user_input)
-        logger.info(f"client_ip: {client_ip}; lang: {lang} params: {url_params}")
-    return text, tokenizer_type_1, tokenizer_type_2

requirements.txt

@@ -1,13 +0,0 @@
-gradio>=4.38.1
-transformers>4.40.0
-sentencepiece
-tiktoken
-icetk
-torch
-nltk
-boto3
-protobuf==4.25.3
-ai2-olmo
-ipadic
-fugashi
-datasets

utils/i18n_util.py

@@ -1,26 +0,0 @@
-import gradio as gr
-
-
-def get_lang(request: gr.Request):
-    """
-    'accept-language', b'zh,en;q=0.9,zh-CN;q=0.8')
-    """
-    accept_language = None
-    langs = []
-    try:
-        accept_language = request.headers["Accept-Language"]
-        for lang in accept_language.split(",")[:5]:
-            lang = lang.lower()
-            if lang.startswith("en"):
-                langs.append("en")
-            elif lang.startswith("es"):
-                langs.append("es")
-            elif lang.startswith("zh"):
-                langs.append("zh")
-            elif lang.startswith("fr"):
-                langs.append("fr")
-            elif lang.startswith("de"):
-                langs.append("de")
-    except Exception as e:
-        print(e)
-    return accept_language, langs

utils/lang_util.py

@@ -1,89 +0,0 @@
-"""
-这个detect_language函数通过定义一系列语言字符的Unicode范围，然后使用regex包来检查输入字符串是否包含这些范围内的字符，
-从而尝试确定字符串可能使用的语言。函数返回一个列表，包含所有匹配的语言名称；如果没有检测到已定义范围内的字符，则返回['Unknown']。
-
-请注意，由于某些语言（如中文和日文）共享字符集的部分范围，这可能导致某些字符串被识别为多种语言。
-此外，Latin范围非常广泛，几乎包括了所有西方语言的基本字母，因此可能需要更细致的逻辑来区分使用拉丁字母的具体语言。
-
-
-通过检查特定的字母和重音符号来区分一些使用拉丁字母的语言。
-然而，需要强调的是，这种方法的准确性受限于所选语言特征的全面性和独特性。
-例如，English的检测范围仅限于基本的A-Z字母，这可能导致它与其他使用相同字母集的语言重叠。
-此外，有些语言（如法语和西班牙语）在某些情况下可能共享特定的重音符号，这可能导致一个字符串被错误地识别为多种语言。
-
-## common language
-English | 简体中文 | 繁體中文 | 한국어 | Español | 日本語 | हिन्दी | Русский | Рortuguês | తెలుగు | Français | Deutsch | Tiếng Việt |
-"""
-
-import re
-from typing import List
-
-# 由于大部分是'latin'，所以就不统计了。
-common_lang = ["Chinese", "Japanese-Kana", "Korean", "Arabic", "number"]
-
-# Unicode range of different language
-language_ranges = {
-    (
-        "Arabic",
-        "ar",
-    ): r"[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF]",
-    # 'CJK'  https://en.wikipedia.org/wiki/CJK_Unified_Ideographs
-    ("Chinese", "zh"): r"[\u4e00-\u9fff]",
-    ("Japanese", "ja"): r"[\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF]",
-    # https://stackoverflow.com/questions/19899554/unicode-range-for-japanese
-    # Kana type refers to Japanese hiragana and katakana characters that represent phonetic sounds in the Japanese language.
-    (
-        "Japanese-Kana",
-        "ja-kana",
-    ): r"[\u3040-\u309F\u30A0-\u30FF]",  # Hiragana  & Katakana
-    ("Korean", "ko"): r"[\uac00-\ud7a3]",
-    # 拉丁字母系列
-    # ('Latin', 'la'): r'[\u0000-\u007F\u0080-\u00FF]',
-    # ('English', 'en'): r'[A-Za-z]',  # 这可能会与其他使用基本拉丁字母的语言重叠
-    # ('French', 'fr'): r'[\u00C0-\u00FF]',
-    # ('German', 'de'): r'[\u00C4\u00D6\u00DC\u00E4\u00F6\u00FC\u00DF]',
-    # ('Spanish-特有'): r'[\u00C1\u00E1\u00C9\u00E9\u00CD\u00ED\u00D3\u00F3\u00DA\u00FA\u00D1\u00F1\u00FC]',  # 西班牙语特有字符集合
-    # 斯拉夫语系列
-    # ('Cyrillic', ''): r'[\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]',
-    #
-    # 'Greek': r'[\u0370-\u03FF\u1F00-\u1FFF]',   # 希腊字母
-    # 'Hebrew': r'[\u0590-\u05FF\uFB1D-\uFB4F]',  # 希伯来语
-}
-
-
-def detect_language_by_unicode(text: str) -> List:
-    """
-    :param text:
-    :return:
-    """
-    detected_languages = []
-    for language, pattern in language_ranges.items():
-        if re.search(pattern, text):
-            detected_languages.append(language)
-
-    return detected_languages
-
-
-if __name__ == "__main__":
-    # 测试函数
-    test_strings = {
-        # 拉丁语系
-        "Hello, world!": "English/Latin",
-        "Hola": "Spanish",
-        "Bonjour": "French",
-        "Guten Tag": "German",
-        "Empieza donde estás. ": "Spanish",
-        # CJK
-        "你好": "Chinese",
-        "こんにちは": "Japanese",
-        "안녕하세요": "Korean",
-        # 其他
-        "Привет": "Russian/Cyrillic",
-        "مرحبا": "Arabic",
-    }
-
-    for s, expected in test_strings.items():
-        # print(f"'{s}' === Detected lang: {detect_language(s)} === Expected: {expected}")
-        print(
-            f"'{s}'\nDetected lang: {detect_language_by_unicode(s)}\nExpected lang: {expected}"
-        )

utils/log_util.py

@@ -1,10 +0,0 @@
-import logging
-
-logging.basicConfig(
-    format="[%(asctime)s] [%(levelname)s] [%(process)d:%(thread)d] [%(filename)s:%(lineno)d:%(funcName)s] %(message)s",
-    level=logging.INFO,
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)

utils/oov_util.py

@@ -1,122 +0,0 @@
-import json
-
-from vocab import TokenizerImpl, all_tokenizer_config, load_tokenizer
-
-text = (
-    "hello; Замглавы управления развития; 특히 주소 15~17번 홀에선 3연속;"
-    " 確実に春が近づいてること;  a közoktatással? _ Belföld;"
-    " pumë, i vjetër, vjeç; ئەردوغان ۋە قىرغىزىستان ;"
-    " निम्न में से कौन सा हारडवेयर; ተለዋዋጭ የግድግዳ ; Дзейныя асобы:;"
-    " « અમરેલીનાં મહિલા વિકાસ; 🦙❤❥웃유♋☮✊;"
-    "װיקיװערטערבוך "
-)
-whitespace = "\t   \n\n\r  "
-bytes = b"\x00\x01\x02\x03\x04".decode("utf-8")
-
-text += whitespace
-
-
-def get_unk(tokenizer_config):
-    tokenizer = load_tokenizer(tokenizer_config)
-    if hasattr(tokenizer, "unk_token"):
-        return f"{tokenizer.unk_token}, {tokenizer.unk_token_id}"
-    else:
-        return "unk_token not found"
-
-
-# def infer_tokenizer_impl(tokenizer_config):
-def infer_tokenizer_type(tokenizer_config):
-    tokenizer = load_tokenizer(tokenizer_config)
-    if tokenizer_config.impl == TokenizerImpl.TikToken:
-        return "tiktoken"
-    if hasattr(tokenizer, "backend_tokenizer"):
-        return str(
-            type(tokenizer.backend_tokenizer.model)
-        )  # type(tokenizer._tokenizer.model))
-    # orion: sp_model.Load(vocab_file)，继承 PreTrainedTokenizer
-    elif hasattr(tokenizer, "sp_model"):  # 基于 sentencepiece 包
-        # for i in range(tokenizer.sp_model.piece_size()):
-        #     if tokenizer.sp_model.is_byte(i):
-        #         print("")
-        return f"sp_model, byte_num: {sum([tokenizer.sp_model.is_byte(i) for i in range(tokenizer.sp_model.piece_size())])}"
-
-    # sp.Load(model_path)  ，并且包括image_tokenizer
-    elif "glm-" in tokenizer_config.name_or_path:
-        return f"byte_num: {sum([tokenizer.sp_tokenizer.text_tokenizer.sp.is_byte(i) for i in range(tokenizer.sp_tokenizer.text_tokenizer.sp.piece_size())])}"
-    # sp.Load(model_path)  ，没有image_tokenizer
-    elif (
-        "glm2-" in tokenizer_config.name_or_path
-        or "glm3-" in tokenizer_config.name_or_path
-        or "CharacterGLM-6B" in tokenizer_config.name_or_path
-    ):
-        return f"byte_num: {sum([tokenizer.tokenizer.sp_model.is_byte(i) for i in range(tokenizer.tokenizer.sp_model.piece_size())])}"
-    elif (
-        "abeja/gpt-neox-japanese-2.7b" == tokenizer_config.name_or_path
-    ):  # 支持 byte-level，解决oov问题
-        return "japanese-bpe: https://github.com/tanreinama/Japanese-BPEEncoder_V2"
-    # bert-base-japanese： 特殊的地方在于 "word_tokenizer_type": "mecab"，见 https://huggingface.co/tohoku-nlp/bert-base-japanese/blob/main/tokenizer_config.json
-    elif "bert-base-japanese" in tokenizer_config.name_or_path:
-        return (
-            "wordpiece.MecabTokenizer, 支持byte-level https://taku910.github.io/mecab/"
-        )
-    elif "moss" in tokenizer_config.name_or_path:
-        return "应该是 sentencepiece.byte_bpe,待确认"
-    elif "byt5" in tokenizer_config.name_or_path:
-        return "未知，待定"
-    else:
-        print("catch", tokenizer_config.name_or_path)
-        raise "error"
-
-
-def test_lossless(tokenizer_config):
-    """
-    xlm-roberta-base 为什么oov这么少？是因为有 byte吗？
-    :param tokenizer_config:
-    :return:
-    """
-    tokenizer = load_tokenizer(tokenizer_config)
-    encoding = tokenizer.encode(text, add_special_tokens=False)
-    decoding = tokenizer.decode(encoding)
-
-    if text in decoding:
-        # print(tokenizer_config.name, tokenizer_config.impl, "lossless: true")
-        pass
-    else:
-        unk_count = sum(
-            [1 for token_id in encoding if token_id == tokenizer.unk_token_id]
-        )
-        oov_tokens = []
-        # if tokenizer_config.impl == TokenizerImpl.SentencePiece:
-        #     print(sum([tokenizer.is_byte(i) for i in range(tokenizer.piece_size())]))
-
-        print("#######" * 5)
-        print(
-            f"{tokenizer_config.name_or_path}, {infer_tokenizer_type(tokenizer_config)}\n"
-            f"lossless: false; unk_token: {get_unk(tokenizer_config)},"
-            f" unk_ratio: {unk_count/len(encoding):.4f}; oov: []"
-        )
-        for i in range(len(text)):
-            if text[i] != decoding[i]:
-                # print(f"text[{i}]     = {str(bytes(text[i:], 'utf-8'))}\n"
-                #       f"decoding[{i}] = {str(bytes(decoding[i:], 'utf-8'))}")
-                print(
-                    f"text[{i}]     = {json.dumps(text[i:], ensure_ascii=False)}, \n"
-                    f"decoding[{i}] = {json.dumps(decoding[i:], ensure_ascii=False)}"
-                )
-
-                break
-
-
-for config in all_tokenizer_config:
-    # if "xlm-roberta-base" in config.name:
-    # if "xlm-roberta-base" in config.name:
-    # if "chatglm3-6b" in config.name:
-    # if "bert-base-japanese" in config.name:
-    # if "moss" in config.name:
-    # if "byt5" in config.name:
-    if "baichuan" in config.name_or_path:
-        # if "CharacterGLM-6B" in config.name:
-        # if "fastchat-t5" in config.name:  # 报错 pyo3_runtime.PanicException: AddedVocabulary bad split
-        # if True:
-        # test_unk(config)
-        test_lossless(config)

utils/text_util.py

@@ -1,47 +0,0 @@
-"""
-char_
-"""
-
-
-def detect_lang_from_unicode():
-    pass
-
-
-def is_digit_char(uchar):
-    return uchar in "0123456789"
-
-
-def contains_digit(text):
-    return any(is_digit_char(ch) for ch in text)
-
-
-def get_digit_count(text):
-    pass
-
-
-def is_all_digit(text):
-    return all(is_digit_char(char) for char in text)
-
-
-def get_digit_count(text):
-    digit_count = 0
-    for char in text:
-        if char in "0123456789":
-            digit_count += 1
-    return digit_count
-
-
-def has_space(text):
-    pass
-
-
-def is_all_space(text):
-    pass
-
-
-def get_space_count(text):
-    space_count = 0
-    for char in text:
-        if len(char.strip()) == 0:
-            space_count += 1
-    return space_count

vocab.py

@@ -1,754 +0,0 @@
-from dataclasses import dataclass, field
-from enum import Enum, auto
-from typing import Any, Dict
-
-import tiktoken
-from transformers import AutoTokenizer
-from utils.log_util import logger
-
-"""Interface:
-# https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py
-
-tokenizer.encode -> List[int]: Converts a string to a sequence of ids (integer)
-tokenizer.decode
-    tokenizer.convert_tokens_to_string   # gpt4 没有这个方法
-tokenizer.convert_ids_to_tokens
-tokenizer.tokenize -> List[str]:  Converts a string into a sequence of tokens ->
-
-
-tokenizer.parent = ""
-tokenizer.vocab_size   
-tokenizer.get_vocab()   # gpt-neox-20b, llama
-tokenizer.type = TokenizerType.ByteBPE.name
-tokenizer.implementation = TokenizerImpl.SentencePiece.name   # https://github.com/facebookresearch/llama/blob/main/llama/tokenizer.py
-  "HFGPT2Tokenizer", "HFTokenizer", "GPT2BPETokenizer", "CharLevelTokenizer", "TiktokenTokenizer", "SPMTokenizer", https://github.com/EleutherAI/gpt-neox/blob/main/tools/preprocess_data.py
-
-    
-tokenizer.comments = "split all numbers into individual digits, " \
-                     "and fallback to bytes to decompose unknown UTF-8 characters"
-
-tokenizer.all_special_tokens  # baichuan
-tokenizer.special_tokens_set   # gpt3.5_turbo
-tokenizer.special_tokens_map   
-"""
-
-
-class TokenizerImpl(Enum):
-    """
-    - https://github.com/huggingface/tokenizers/blob/main/bindings/python/py_src/tokenizers/implementations/__init__.py
-    - https://huggingface.co/docs/transformers/tokenizer_summary
-    - https://github.com/EleutherAI/gpt-neox/blob/main/megatron/tokenizer/tokenizer.py
-
-    ## google/BertTokenizer
-    - https://github.com/huggingface/tokenizers/blob/main/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
-    - 特征
-        - 算法：BERT的编码器是 BPE-WordPiece，将单词拆分成多个前缀符号（比如BERT中的##）最小单元
-        - 词典：有##开头的token，表示subword，
-            - 中文采用char粒度分词
-            - 英文采用  WordPiece
-
-
-
-
-    ## google/sentencepiece
-    - https://github.com/google/sentencepiece/blob/3863f7648e5d8edb571ac592f3ac4f5f0695275a/src/sentencepiece_model.proto#L48
-    - 支持 sentencepiece 和 wordpiece
-        - sentencepiece 有byte-bpe吗？
-            - UNIGRAM = 1;  // Unigram language model with dynamic algorithm
-            - BPE = 2;      // Byte Pair Encoding
-            - WORD = 3;     // Delimitered by whitespace.
-            - CHAR = 4;     // tokenizes into character sequence
-        - wordpiece
-    - 特征：
-        - 训练: spm_train --model_type unigram/bpe/char/word
-        - 特殊符号： Ġ
-        - 文件: *.sp_model  或 *.model  (可选文件 .vocab，) spm简称   (其他格式比如 tokenizer.json是给hf_tokenizer兼容用的)
-        - 实现:
-            - 依赖: protobuf
-            - 训练: `import sentencepiece as spm; spm.SentencePieceTrainer.train` 或 `spm_train`
-            - 加载: `import sentencepiece as spm; spm.SentencePieceProcessor().Load(vocab_file)`
-            - 方法: 是SentencePieceProcessor类型，sp_model.id_to_piece，有tokenizer.json tokenizer.model，
-            - 分词:
-                - pre_tokenizers.ByteLevel(add_prefix_space=True, use_regex=False)
-        - 词典:  词典字符有 ▁  (U+2581) ，表示空格或句首。
-    - 示例：google-t5, llama，baichuan, orion,
-        - llama: tokenizer.json(包含model.vocab model.merges)  tokenizer.model
-        - grok: 原始是 .model文件，后面转成了 tokenizer.json
-        - google-t5: tokenizer.json, spiece.model
-        - Skywork-13B-Math: tokenizer.model
-        - xlm_roberta: sentencepiece.bpe.model
-        - GPT2Tokenizer
-            - tokenizer.json, vocab.json, merges.txt   (https://huggingface.co/openai-community/gpt2)
-            - vocab.bpe, encoder.json, dict.txt  （fairseq版本，不常用，可以忽略这个版本）
-
-
-
-    ## thu/icetk
-      - icetk： sentencepiece的分支，支持image_tokenizer。
-    - glm, chatglm1, chatglm2
-
-    ## huggingface/tokenizers
-    - https://github.com/huggingface/tokenizers
-    - VS sentencepiece
-        - 支持sentencepiece
-            - .model转化为 (merges.txt + vocab.json) 或者 tokenizer.json
-                - https://github.com/huggingface/tokenizers/blob/main/bindings/python/scripts/sentencepiece_extractor.py
-            - 加载 merges.txt, vocab.json
-                - SentencePieceBPETokenizer  https://github.com/huggingface/tokenizers/blob/v0.19.1/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py#L10
-        - 在 sentencepiece基础上，hf_tokenizer支持pre-tokenization的正则表达式，对tab和换行支持更好，支持special token
-    - 类型： 支持 BBPE, WordPiece or Unigram
-    - 特征：
-        - 文件: tokenizer.json(包含后两个文件的内容), merges.txt, vocab.json
-            - added_tokens 在vocab中不一定存在。
-        - 实现:
-            - 训练: `from tokenizers.trainers import BpeTrainer, UnigramTrainer, WordLevelTrainer, WordPieceTrainer`
-            - 加载:
-            - 方法: .model.from_file  .model.save   .model.token_to_id  .model.tokenize
-        - .model 是 tokenizer.models.BPE 类型
-        - 词典有 Ġ  "\u0120" 开头
-        - 优势
-        -
-    - 示例：gpt2, gpt_neox_20b, moss, bloom, qwen2
-    - 优势：相对sentence piece，
-        - ss
-
-    ## openai/tiktoken
-    - 特征：空格就是空格，
-    - 示例：gpt3.5 gpt4, qwen,
-    """
-
-    """ 算法体系  https://www.huaxiaozhuan.com/%E5%B7%A5%E5%85%B7/huggingface_transformer/chapters/1_tokenizer.html
-    - word-base tokenizer:
-    - char-base tokenizer:
-    - subword-based Tokenizer
-        - BPE 
-            - byte-bpe: base vocabulary大小是256
-        - WordPiece:
-            - 相比BPE，WordPiece 仅保存最终词表，而不保存学到的 merge rule
-        - Unigram
-    - SentencePiece
-    
-    """
-
-    # 分类体系：https://github.com/huggingface/tokenizers/blob/main/bindings/python/py_src/tokenizers/implementations/
-    BertTokenizer = "wordpiece.BertTokenizer"
-    JapaneseTokenizer = (
-        "wordpiece.MecabTokenizer",
-        "https://github.com/polm/fugashi",
-    )  # 常用日语包 ipadic，fugashi，
-    ByteLevelBPETokenizer = "byte_level_bpe"  # BBPE
-    SentencePieceBPETokenizer = "sentencepiece_bpe"
-
-    # 分类体系
-
-    # SentencePeice(BPE)
-    SentencePiece = auto()  # sentencepiece.bpe, sentencepiece.unigram, sentencepiece.char, sentencepiece.word,
-    byte_level_bpe = auto()
-    # HFTokenizer = auto()  # , 支持
-    TikToken = auto()
-    # subword-nmt
-    # WordPiece
-
-
-# load_vocab_with_SPECIAL_TOKEN = True # 如果不包含会导致计算词典大小错误、overlap_token计算不一致。
-
-
-@dataclass
-class TokenizerConfig:
-    """
-    https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/leaderboard/read_evals.py
-    """
-
-    name_or_path: str  # org/model (path on hub), as unique id
-    name_display: str = None  #
-    impl: TokenizerImpl = None  # implementation, tokenizer_class/type
-    org: str = None
-    link: str = None  # http://**
-    desc: str = None  # description
-    meta: str = None
-    level: str = None  # char-level, word-level, byte-level
-    lang: str = None
-    init_kwargs: Dict[str, Any] = field(
-        default_factory=dict,
-    )
-
-    def __post_init__(self):
-        if self.link is None:
-            self.link = "https://huggingface.co/" + self.name_or_path  # TODO + revision
-        if self.name_display is None:
-            self.name_display = self.name_or_path
-
-    @classmethod
-    def init_from_json_file(cls, json_filepath: str) -> "TokenizerConfig":
-        pass
-
-    def __eq__(self, other):
-        if isinstance(other, self.__class__):
-            return self.__dict__ == other.__dict__
-        else:
-            return False
-
-    def __hash__(self):
-        return hash(self.name_or_path)
-
-
-# TODO: append link and description to the end of dropdown button.
-# Add tokenizer_class/type, comments
-_all_tokenizer_config = [
-    # bert style tokenizers
-    TokenizerConfig(
-        "google-bert/bert-base-cased",
-        impl=TokenizerImpl.BertTokenizer,
-        org="Google",
-        desc="first add whitespace around any CJK character, then perform wordpiece tokenization.",
-    ),
-    TokenizerConfig(
-        "google-bert/bert-base-uncased",
-        impl=TokenizerImpl.BertTokenizer,
-        org="Google",
-        desc="first add whitespace around any CJK character, then perform wordpiece tokenization.",
-    ),
-    TokenizerConfig(
-        "google-bert/bert-base-chinese",
-        impl=TokenizerImpl.BertTokenizer,
-        org="Google",
-        desc="first add whitespace around any CJK character, then perform wordpiece tokenization.",
-    ),
-    TokenizerConfig(
-        "google-bert/bert-base-german-cased",
-        impl=TokenizerImpl.BertTokenizer,
-        org="Google",
-    ),
-    TokenizerConfig(
-        "dbmdz/bert-base-german-uncased", impl=TokenizerImpl.BertTokenizer, org="dbmdz"
-    ),
-    TokenizerConfig(
-        "asafaya/bert-base-arabic", impl=TokenizerImpl.BertTokenizer, org="-"
-    ),
-    TokenizerConfig(
-        "google-bert/bert-base-multilingual-uncased",
-        impl=TokenizerImpl.BertTokenizer,
-        org="Google",
-    ),
-    TokenizerConfig(
-        "google-bert/bert-base-multilingual-cased",
-        impl=TokenizerImpl.BertTokenizer,
-        org="Google",
-    ),
-    TokenizerConfig(
-        "tohoku-nlp/bert-base-japanese",
-        impl=TokenizerImpl.BertTokenizer,
-        org="Tohoku",
-        desc="The texts are first tokenized by MeCab morphological parser with the IPA dictionary, "
-        "then split into subwords by the WordPiece algorithm.",
-    ),
-    TokenizerConfig(
-        "clue/roberta_chinese_clue_tiny",
-        name_display="clue/roberta-chinese-clue",
-        impl=TokenizerImpl.BertTokenizer,
-        org="CLUE",
-        init_kwargs={"revision": "refs/pr/1"},
-        desc="",
-        meta="去掉了繁体字, https://github.com/CLUEbenchmark/CLUEPretrainedModels/blob/master/README.md",
-    ),
-    TokenizerConfig(
-        "eson/kplug-base-encoder",
-        name_display="eson/kplug",
-        impl=TokenizerImpl.BertTokenizer,
-        org="JD",
-    ),
-    TokenizerConfig(
-        "ckiplab/gpt2-base-chinese", impl=TokenizerImpl.BertTokenizer, org="SINICA"
-    ),  # 台湾中央研究院
-    # WoBERT  https://kexue.fm/archives/7758
-    # WoBERT Plus  https://github.com/ZhuiyiTechnology/WoBERT
-    # gpt2 style tokenizers
-    TokenizerConfig(
-        "openai-community/gpt2", impl=TokenizerImpl.SentencePiece, org="OpenAI"
-    ),
-    # byte-level BPE,没有byte，是unicode-level的吗？
-    TokenizerConfig(
-        "ClassCat/gpt2-base-french", impl=TokenizerImpl.SentencePiece, org="ClassCat"
-    ),
-    TokenizerConfig(
-        "ClassCat/gpt2-base-spanish", impl=TokenizerImpl.SentencePiece, org="ClassCat"
-    ),
-    TokenizerConfig(
-        "fnlp/moss-moon-003-sft",
-        impl=TokenizerImpl.SentencePiece,
-        init_kwargs={"revision": "refs/pr/6"},
-        org="Fudan",
-        desc="This tokenizer has been trained to treat spaces like parts of the tokens "
-        "(a bit like sentencepiece) so a word will be encoded differently whether "
-        "it is at the beginning of the sentence (without space) or not",
-        meta="在gpt2词典基础上，扩充了5万中文",
-    ),
-    TokenizerConfig(
-        "bigscience/bloom",
-        impl=TokenizerImpl.SentencePiece,
-        org="BigScience",
-        meta="比gpt_neox的词典 对中文支持更好。",
-    ),
-    # ("bloomz_6b4_zh",
-    # ("BelleGroup/BELLE-7B-2M",   # 模型和词典都基于bloom
-    #
-    TokenizerConfig(
-        "EleutherAI/gpt-neox-20b", impl=TokenizerImpl.SentencePiece, org="EleutherAI"
-    ),  # 5万
-    TokenizerConfig(
-        "cyberagent/open-calm-7b", impl=TokenizerImpl.SentencePiece, org="CyberAgent"
-    ),  # GPTNeoXTokenizer
-    TokenizerConfig(
-        "abeja/gpt-neox-japanese-2.7b", impl=TokenizerImpl.SentencePiece, org="ABEJA"
-    ),
-    TokenizerConfig(
-        "rinna/bilingual-gpt-neox-4b",
-        impl=TokenizerImpl.SentencePiece,
-        org="ABEJA",
-        lang="en/ja",
-    ),
-    TokenizerConfig(
-        "Qwen/Qwen1.5-14B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
-    ),  # 15万，速度有点慢
-    TokenizerConfig(
-        "Qwen/Qwen1.5-110B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
-    ),
-    TokenizerConfig(
-        "Qwen/Qwen1.5-1.8B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
-    ),
-    TokenizerConfig("Qwen/Qwen2-0.5B", impl=TokenizerImpl.SentencePiece, org="Alibaba"),
-    TokenizerConfig("Qwen/Qwen2-72B", impl=TokenizerImpl.SentencePiece, org="Alibaba"),
-    TokenizerConfig(
-        "Qwen/Qwen2.5-0.5B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
-    ),
-    TokenizerConfig(
-        "Qwen/Qwen2.5-72B", impl=TokenizerImpl.SentencePiece, org="Alibaba"
-    ),
-    TokenizerConfig(
-        "HuggingFaceH4/starchat-alpha", impl=TokenizerImpl.SentencePiece, org="-"
-    ),
-    ####### google/sentencepiece tokenizer:
-    # T5 llama internlm
-    TokenizerConfig(
-        "google-t5/t5-large",
-        name_display="google-t5/t5",
-        impl=TokenizerImpl.SentencePiece,
-        org="Google",
-    ),
-    # t5_small, t5_base, t5_large, flan_t5_base,
-    # ("t5_base", "", "sentencepiece"),
-    # TokenizerConfig("google/flan-t5-base", impl=TokenizerImpl.SentencePiece, ),
-    TokenizerConfig(
-        "lmsys/fastchat-t5-3b-v1.0",
-        impl=TokenizerImpl.SentencePiece,
-        org="LMSYS",
-        init_kwargs={
-            "use_fast": False
-        },  # 解决 pyo3_runtime.PanicException: AddedVocabulary bad split
-    ),
-    TokenizerConfig(
-        "CohereForAI/aya-101", org="Cohere For AI"
-    ),  # "tokenizer_class": "T5Tokenizer",
-    TokenizerConfig(
-        "ClueAI/ChatYuan-large-v2", impl=TokenizerImpl.SentencePiece, org="CLUE"
-    ),
-    TokenizerConfig(
-        "ClueAI/PromptCLUE-base", impl=TokenizerImpl.SentencePiece, org="CLUE"
-    ),
-    # byte-level BPE
-    # '中文单字': 700, '中文多字': 0  meta-llama/Meta-Llama-3.1-405B
-    #
-    TokenizerConfig(
-        "meta-llama/Llama-3.2-1B-Instruct", impl=TokenizerImpl.SentencePiece, org="Meta"
-    ),
-    TokenizerConfig(
-        "meta-llama/Llama-3.2-3B-Instruct", impl=TokenizerImpl.SentencePiece, org="Meta"
-    ),
-    # TokenizerConfig("meta-llama/Llama-3.3-70B-Instruct", impl=TokenizerImpl.SentencePiece,
-    #                 org="Meta"),
-    TokenizerConfig(
-        "meta-llama/Meta-Llama-3.1-405B", impl=TokenizerImpl.SentencePiece, org="Meta"
-    ),
-    TokenizerConfig(
-        "NousResearch/Hermes-3-Llama-3.1-405B",
-        impl=TokenizerImpl.SentencePiece,
-        org="NousResearch",
-    ),
-    TokenizerConfig(
-        "gradientai/Llama-3-8B-Instruct-Gradient-1048k",
-        name_display="Meta/llama3",
-        impl=TokenizerImpl.SentencePiece,
-        org="Meta",
-        desc="llama split all numbers into individual digits, and fallback to bytes to decompose unknown UTF-8 characters",
-    ),
-    TokenizerConfig(
-        "NousResearch/Llama-2-7b-chat-hf",
-        name_display="Meta/llama2",
-        impl=TokenizerImpl.SentencePiece,
-        org="Meta",
-    ),
-    TokenizerConfig(
-        "huggyllama/llama-7b",
-        name_display="Meta/llama",
-        impl=TokenizerImpl.SentencePiece,
-        org="Meta",
-    ),
-    TokenizerConfig(
-        "hpcai-tech/grok-1",
-        name_display="xai-org/grok-1",
-        impl=TokenizerImpl.SentencePiece,
-        org="xAI",
-    ),
-    # 由.model文件转化为了
-    TokenizerConfig(
-        "hfl/chinese-llama-lora-7b",
-        impl=TokenizerImpl.SentencePiece,
-        org="-",
-        meta="向原始LLaMA的词汇表中添加2w个中文词汇，针对原版LLaMA模型扩充了中文词表， 提升了中文编解码效率",
-    ),
-    #
-    TokenizerConfig(
-        "hfl/chinese-llama-2-7b",
-        impl=TokenizerImpl.SentencePiece,
-        org="-",
-        meta="重新设计了新词表（大小：55296），进一步提升了中文字词的覆盖程度",
-    ),  #
-    TokenizerConfig(
-        "hfl/llama-3-chinese-8b", impl=TokenizerImpl.SentencePiece, org="-"
-    ),
-    TokenizerConfig(
-        "hfl/chinese-alpaca-lora-7b", impl=TokenizerImpl.SentencePiece, org="-"
-    ),
-    # 中文Alpaca模型在上述中文LLaMA模型的基础上进一步使用了指令数据进行精调。  "比chinese_llama词典多一个`[PAD]`，请勿混用"
-    #
-    # ("belle_llama_ext_7b",
-    # ("alpaca_7b",
-    TokenizerConfig(
-        "baichuan-inc/Baichuan-7B",
-        name_display="baichuan-inc/baichuan",
-        impl=TokenizerImpl.SentencePiece,
-        level="byte-level",
-        org="Baichuan",
-    ),
-    TokenizerConfig(
-        "baichuan-inc/Baichuan2-7B-Chat",
-        name_display="baichuan-inc/baichuan2",
-        impl=TokenizerImpl.SentencePiece,
-        org="Baichuan",
-        desc="expand the vocabulary size from 64000 in Baichuan1 to 125696",
-    ),
-    TokenizerConfig(
-        "internlm/internlm-chat-7b",
-        impl=TokenizerImpl.SentencePiece,
-        org="Shanghai AI Lab",
-    ),
-    # 上海AI实验室 +  商汤
-    TokenizerConfig(
-        "internlm/internlm2-chat-7b",
-        impl=TokenizerImpl.SentencePiece,
-        org="Shanghai AI Lab",
-    ),
-    TokenizerConfig(
-        "internlm/internlm2-math-7b",
-        impl=TokenizerImpl.SentencePiece,
-        org="Shanghai AI Lab",
-    ),
-    TokenizerConfig(
-        "internlm/internlm-xcomposer-7b",
-        impl=TokenizerImpl.SentencePiece,
-        org="Shanghai AI Lab",
-    ),
-    TokenizerConfig("tiiuae/falcon-7b", impl=TokenizerImpl.SentencePiece, org="TII"),
-    TokenizerConfig("tiiuae/falcon-180b", impl=TokenizerImpl.SentencePiece, org="TII"),
-    TokenizerConfig(
-        "Skywork/Skywork-13B-base", impl=TokenizerImpl.SentencePiece, org="Kunlun"
-    ),
-    TokenizerConfig(
-        "Skywork/Skywork-13B-Math", impl=TokenizerImpl.SentencePiece, org="Kunlun"
-    ),  # 文件：tokenizer.model
-    TokenizerConfig(
-        "FacebookAI/xlm-roberta-base", impl=TokenizerImpl.SentencePiece, org="Facebook"
-    ),
-    # 这个的tokenizer.json 为什么没有merges? vocab里为什么有概率值？
-    # "goat",
-    # ##### glm系列
-    # "glm_chinese",),
-    TokenizerConfig(
-        "THUDM/chatglm-6b",
-        impl=TokenizerImpl.SentencePiece,
-        org="Tsinghua",
-        meta=f"num_image_tokens: {12}; num_image_tokens: {34} ",
-        init_kwargs={"revision": "refs/pr/100"},
-    ),
-    TokenizerConfig(
-        "THUDM/chatglm2-6b",
-        impl=TokenizerImpl.SentencePiece,
-        org="Tsinghua",
-    ),
-    TokenizerConfig(
-        "THUDM/chatglm3-6b",
-        impl=TokenizerImpl.SentencePiece,
-        org="Tsinghua",
-    ),
-    TokenizerConfig(
-        "thu-coai/CharacterGLM-6B",
-        impl=TokenizerImpl.SentencePiece,
-        org="Tsinghua",
-    ),
-    # tiktoken 系列
-    TokenizerConfig(
-        "openai/text-davinci-003",
-        impl=TokenizerImpl.TikToken,
-        org="OpenAI",
-        link="https://github.com/openai/tiktoken",
-    ),
-    #
-    TokenizerConfig(
-        "openai/code-davinci-002",
-        impl=TokenizerImpl.TikToken,
-        org="OpenAI",
-        link="https://github.com/openai/tiktoken",
-    ),
-    TokenizerConfig(
-        "openai/gpt-3.5-turbo",
-        impl=TokenizerImpl.TikToken,
-        org="OpenAI",
-        link="https://github.com/openai/tiktoken",
-        desc="tiktoken is a fast BPE tokeniser for use with OpenAI's models. There are 16 tokens KeyError",
-    ),
-    TokenizerConfig(
-        "openai/gpt-4",
-        impl=TokenizerImpl.TikToken,
-        org="OpenAI",
-        link="https://github.com/openai/tiktoken",
-    ),
-    TokenizerConfig(
-        "openai/gpt-4o",
-        impl=TokenizerImpl.TikToken,
-        org="OpenAI",
-        link="https://github.com/openai/tiktoken",
-    ),
-    TokenizerConfig(
-        "Qwen/Qwen-7B-Chat",
-        name_display="Qwen/Qwen",
-        impl=TokenizerImpl.TikToken,
-        org="Alibaba",
-        init_kwargs={"revision": "refs/pr/56"},
-        meta="在gpt4词典基础上，删除了100个多数字token，增加10000中文词token；并优化了special_token的分词",
-    ),
-    # https://huggingface.co/Qwen/Qwen-7B-Chat#%E6%A8%A1%E5%9E%8B%E7%BB%86%E8%8A%82%EF%BC%88model%EF%BC%89
-    #  该词表在GPT-4使用的BPE词表cl100k_base基础上，对中文、多语言进行了优化，在对中、英、代码数据的高效编解码的基础上，
-    #  对部分多语言更加友好，方便用户在不扩展词表的情况下对部分语种进行能力增强。 词表对数字按单个数字位切分。
-    # TokenizerConfig("Qwen/Qwen-72B-Chat", impl=TokenizerImpl.TikToken),
-    # 未分类
-    # ("amber", ""),
-    TokenizerConfig("LLM360/CrystalCoder", org="MBZUAI"),
-    TokenizerConfig("apple/DCLM-7B", org="Apple"),
-    TokenizerConfig("mistralai/Mistral-7B-v0.1", org="Mistral"),
-    TokenizerConfig("mistralai/Mixtral-8x7B-v0.1", org="Mistral"),
-    TokenizerConfig("mistralai/Mistral-Large-Instruct-2407", org="Mistral"),
-    TokenizerConfig("mistralai/Mistral-Nemo-Instruct-2407", org="Mistral"),
-    TokenizerConfig("paust/pko-t5-large", org="PAUST"),
-    TokenizerConfig("01-ai/Yi-6B", org="Yi"),
-    TokenizerConfig("01-ai/Yi-34B", org="Yi"),
-    TokenizerConfig("01-ai/Yi-VL-34B", org="Yi"),
-    TokenizerConfig("01-ai/Yi-1.5-34B", org="Yi"),
-    TokenizerConfig("OrionStarAI/Orion-14B-Chat", org="OrionStar"),
-    TokenizerConfig("microsoft/phi-1", org="Microsoft"),
-    TokenizerConfig("microsoft/phi-2", org="Microsoft"),
-    TokenizerConfig(
-        "microsoft/Phi-3-mini-4k-instruct", org="Microsoft", meta="即llama vocab"
-    ),
-    TokenizerConfig("Upstage/SOLAR-10.7B-v1.0", org="-"),
-    TokenizerConfig("google/mobilebert-uncased", org="Google"),
-    # ("google/mobilenet_v2_1.0_224",),  # error
-    TokenizerConfig("google/switch-c-2048", org="Google"),
-    TokenizerConfig("google/byt5-small", org="Google"),
-    TokenizerConfig("google/mt5-large", org="Google"),
-    TokenizerConfig("WizardLM/WizardCoder-Python-7B-V1.0", org="Microsoft"),
-    TokenizerConfig("WizardLM/WizardCoder-15B-V1.0", org="Microsoft"),
-    TokenizerConfig("WizardLM/WizardLM-7B-V1.0", org="Microsoft"),
-    TokenizerConfig("WizardLM/WizardMath-70B-V1.0", org="Microsoft"),
-    TokenizerConfig("TigerResearch/tigerbot-70b-chat-v4-4k", org="Tigerobo"),
-    TokenizerConfig("TigerResearch/tigerbot-13b-chat-v2", org="Tigerobo"),
-    TokenizerConfig("deepseek-ai/deepseek-coder-33b-instruct", org="DeepSeek"),
-    TokenizerConfig("deepseek-ai/deepseek-llm-7b-base", org="DeepSeek"),
-    TokenizerConfig("deepseek-ai/DeepSeek-V2", org="DeepSeek"),
-    TokenizerConfig("deepseek-ai/DeepSeek-V3", org="DeepSeek"),
-    TokenizerConfig(
-        "deepseek-ai/DeepSeek-R1", org="DeepSeek"
-    ),  # 在llama3的词典上，增加了一些中文token，删掉了一部分token
-    TokenizerConfig("deepseek-ai/DeepSeek-R1-Zero", org="DeepSeek"),
-    TokenizerConfig("deepseek-ai/DeepSeek-R1-Distill-Llama-70B", org="DeepSeek"),
-    TokenizerConfig("google/gemma-7b", org="Google"),
-    TokenizerConfig("google/gemma-2-9b", org="Google"),
-    TokenizerConfig("allenai/OLMo-7B-hf", org="Allen AI"),
-    TokenizerConfig("HuggingFaceH4/zephyr-7b-beta", org="HuggingFace"),
-    TokenizerConfig("ai21labs/Jamba-v0.1", org="AI21"),
-    TokenizerConfig("databricks/dbrx-instruct", org="Databricks"),
-    TokenizerConfig("MiniMaxAI/MiniMax-Text-01", org="MiniMax"),
-    # TokenizerConfig("nvidia/Nemotron-4-340B-Instruct", org="Nvidia"),
-    # ("claude",),
-    # https://github.com/Duxiaoman-DI/XuanYuan
-    # https://huggingface.co/apple/OpenELM-3B-Instruct  https://huggingface.co/apple/OpenELM-3B
-]
-
-assert len(set([config.name_display for config in _all_tokenizer_config])) == len(
-    _all_tokenizer_config
-)
-assert len(set([config.name_or_path for config in _all_tokenizer_config])) == len(
-    _all_tokenizer_config
-)
-assert len(
-    set([config.name_or_path.split("/")[-1] for config in _all_tokenizer_config])
-) == len(_all_tokenizer_config)
-
-
-class TokenizerFactory:
-    def __init__(self):
-        # self.all_tokenizer_configs = sorted(_all_tokenizer_config, key=lambda k: k.name_or_path)
-        self.all_tokenizer_configs = sorted(
-            _all_tokenizer_config, key=lambda k: k.name_display
-        )
-        self.all_tokenizer_names = [
-            config.name_or_path for config in self.all_tokenizer_configs
-        ]
-        self.name_to_config_list = [
-            {config.name_or_path: config for config in self.all_tokenizer_configs},
-            {config.name_display: config for config in self.all_tokenizer_configs},
-            {
-                config.name_display.split("/")[-1]: config
-                for config in self.all_tokenizer_configs
-            },
-        ]
-        self.tokenizer_cache = {}
-
-    def get_tokenizer_config(self, tokenizer_name: str) -> TokenizerConfig:
-        for name_to_config in self.name_to_config_list:
-            if tokenizer_name in name_to_config:
-                return name_to_config[tokenizer_name]
-        return None
-
-    def get_tokenizer(self, tokenizer_name: str):
-        """
-        :param tokenizer_name:
-        :return:
-        """
-        tokenizer_config = self.get_tokenizer_config(tokenizer_name)
-
-        # 1. load from cache
-        if tokenizer_config in self.tokenizer_cache:
-            return self.tokenizer_cache[tokenizer_config]
-
-        # 2. load tokenizer
-        tokenizer = self.load_tokenizer(tokenizer_config)
-
-        self.tokenizer_cache[tokenizer_config] = tokenizer
-        return tokenizer
-
-    def get_name_with_hyperlink(self, tokenizer_name: str) -> str:
-        def model_hyperlink(link, model_name):
-            model_name = model_name
-            return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-        tokenizer_config = self.get_tokenizer_config(tokenizer_name)
-        return model_hyperlink(
-            tokenizer_config.link, tokenizer_config.name_display.split("/")[-1]
-        )
-
-    def load_tokenizer(self, tokenizer_config):
-        if tokenizer_config == None:
-            print("dd")
-        logger.info(f"loading tokenizer {tokenizer_config.name_or_path}")
-        if (
-            tokenizer_config.impl == TokenizerImpl.TikToken
-            and "openai" in tokenizer_config.name_or_path
-        ):
-            tokenizer = tiktoken.encoding_for_model(
-                tokenizer_config.name_or_path.replace("openai/", "")
-            )
-        else:
-            tokenizer = AutoTokenizer.from_pretrained(
-                tokenizer_config.name_or_path,
-                trust_remote_code=True,
-                **tokenizer_config.init_kwargs,
-            )
-        return tokenizer
-
-    def add_config(
-        self,
-    ):
-        pass
-
-    def add_tokenizer(self, tokenizer_name):
-        pass
-
-
-tokenizer_factory = TokenizerFactory()
-
-
-def add_tokenizer(tokenizer_name: str):
-    """
-    :param tokenizer_name:
-    :return:
-    """
-    if tokenizer_name in []:
-        logger.info(f"{tokenizer_name} already exits")
-    else:
-        # add to config
-        tokenizer_config = TokenizerConfig(tokenizer_name, org="-")
-
-        # add to tokenizer
-        tokenizer = tokenizer_factory.load_tokenizer(tokenizer_config)
-
-        # refresh cache
-
-        try:
-            tokenizer = AutoTokenizer.from_pretrained(
-                tokenizer_name, trust_remote_code=True, **tokenizer_config.init_kwargs
-            )
-            tokenizer_factory.all_tokenizer_configs.append(
-                "",
-            )
-            tokenizer_factory
-
-        except Exception as e:
-            logger.error(e)
-
-    pass
-
-
-# class TokenizerType(Enum):
-#
-#     # BERTTokenizer
-#     # 依赖一个txt文件
-#
-#
-#     # https://github.com/EleutherAI/gpt-neox/blob/v2.0/megatron/tokenizer/tokenizer.py#L231
-#     # 依赖一个json文件，Tokenizer.from_file(vocab_file)
-#     # 案例：gpt-neox-20B
-#     HFTokenizer = auto()
-#
-#     # 依赖: model_file, sentencepiece.SentencePieceProcessor(model_file)
-#     # 案例：
-#     SentencePieceTokenizer = auto()
-#
-#
-#     # 依赖: 3个json文件：vocab.json, merges.txt, special_tokens.txt
-#     # 源码:
-#     #   - https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/tokenizer/gpt2_tokenization.py#L92
-#     # Byte-level BPE
-#     GPT2BPETokenizer = auto()
-
-
-if __name__ == "__main__":
-    for tokenizer_config in tokenizer_factory.all_tokenizer_configs:
-        if True:
-            # if "t5" in tokenizer_config.name_or_path:
-            tokenizer1 = tokenizer_factory.get_tokenizer(tokenizer_config.name_or_path)
-            tokenizer2 = tokenizer_factory.get_tokenizer(tokenizer_config.name_display)
-            tokenizer3 = tokenizer_factory.get_tokenizer(
-                tokenizer_config.name_display.split("/")[-1]
-            )
-            assert tokenizer1 == tokenizer2 == tokenizer3
-            print(tokenizer_config.name_or_path, len(tokenizer1))