relik/inference/data/tokenizers/__init__.py

SPACY_LANGUAGE_MAPPER = {
    "ca": "ca_core_news_sm",
    "da": "da_core_news_sm",
    "de": "de_core_news_sm",
    "el": "el_core_news_sm",
    "en": "en_core_web_sm",
    "es": "es_core_news_sm",
    "fr": "fr_core_news_sm",
    "it": "it_core_news_sm",
    "ja": "ja_core_news_sm",
    "lt": "lt_core_news_sm",
    "mk": "mk_core_news_sm",
    "nb": "nb_core_news_sm",
    "nl": "nl_core_news_sm",
    "pl": "pl_core_news_sm",
    "pt": "pt_core_news_sm",
    "ro": "ro_core_news_sm",
    "ru": "ru_core_news_sm",
    "xx": "xx_sent_ud_sm",
    "zh": "zh_core_web_sm",
    "ca_core_news_sm": "ca_core_news_sm",
    "ca_core_news_md": "ca_core_news_md",
    "ca_core_news_lg": "ca_core_news_lg",
    "ca_core_news_trf": "ca_core_news_trf",
    "da_core_news_sm": "da_core_news_sm",
    "da_core_news_md": "da_core_news_md",
    "da_core_news_lg": "da_core_news_lg",
    "da_core_news_trf": "da_core_news_trf",
    "de_core_news_sm": "de_core_news_sm",
    "de_core_news_md": "de_core_news_md",
    "de_core_news_lg": "de_core_news_lg",
    "de_dep_news_trf": "de_dep_news_trf",
    "el_core_news_sm": "el_core_news_sm",
    "el_core_news_md": "el_core_news_md",
    "el_core_news_lg": "el_core_news_lg",
    "en_core_web_sm": "en_core_web_sm",
    "en_core_web_md": "en_core_web_md",
    "en_core_web_lg": "en_core_web_lg",
    "en_core_web_trf": "en_core_web_trf",
    "es_core_news_sm": "es_core_news_sm",
    "es_core_news_md": "es_core_news_md",
    "es_core_news_lg": "es_core_news_lg",
    "es_dep_news_trf": "es_dep_news_trf",
    "fr_core_news_sm": "fr_core_news_sm",
    "fr_core_news_md": "fr_core_news_md",
    "fr_core_news_lg": "fr_core_news_lg",
    "fr_dep_news_trf": "fr_dep_news_trf",
    "it_core_news_sm": "it_core_news_sm",
    "it_core_news_md": "it_core_news_md",
    "it_core_news_lg": "it_core_news_lg",
    "ja_core_news_sm": "ja_core_news_sm",
    "ja_core_news_md": "ja_core_news_md",
    "ja_core_news_lg": "ja_core_news_lg",
    "ja_dep_news_trf": "ja_dep_news_trf",
    "lt_core_news_sm": "lt_core_news_sm",
    "lt_core_news_md": "lt_core_news_md",
    "lt_core_news_lg": "lt_core_news_lg",
    "mk_core_news_sm": "mk_core_news_sm",
    "mk_core_news_md": "mk_core_news_md",
    "mk_core_news_lg": "mk_core_news_lg",
    "nb_core_news_sm": "nb_core_news_sm",
    "nb_core_news_md": "nb_core_news_md",
    "nb_core_news_lg": "nb_core_news_lg",
    "nl_core_news_sm": "nl_core_news_sm",
    "nl_core_news_md": "nl_core_news_md",
    "nl_core_news_lg": "nl_core_news_lg",
    "pl_core_news_sm": "pl_core_news_sm",
    "pl_core_news_md": "pl_core_news_md",
    "pl_core_news_lg": "pl_core_news_lg",
    "pt_core_news_sm": "pt_core_news_sm",
    "pt_core_news_md": "pt_core_news_md",
    "pt_core_news_lg": "pt_core_news_lg",
    "ro_core_news_sm": "ro_core_news_sm",
    "ro_core_news_md": "ro_core_news_md",
    "ro_core_news_lg": "ro_core_news_lg",
    "ru_core_news_sm": "ru_core_news_sm",
    "ru_core_news_md": "ru_core_news_md",
    "ru_core_news_lg": "ru_core_news_lg",
    "xx_ent_wiki_sm": "xx_ent_wiki_sm",
    "xx_sent_ud_sm": "xx_sent_ud_sm",
    "zh_core_web_sm": "zh_core_web_sm",
    "zh_core_web_md": "zh_core_web_md",
    "zh_core_web_lg": "zh_core_web_lg",
    "zh_core_web_trf": "zh_core_web_trf",
}

from relik.inference.data.tokenizers.spacy_tokenizer import SpacyTokenizer