line-corporation
/

line-distilbert-base-japanese

make `tokenizer_class` same as the actual class name

by tmanabe - opened Nov 24

←

Files changed (1) hide show

tokenizer_config.json CHANGED Viewed

@@ -17,7 +17,7 @@
        "__type":"AddedToken"
     },
     "tokenize_chinese_chars":false,
-    "tokenizer_class": "BertJapaneseTokenizer",
     "word_tokenizer_type": "mecab",
     "subword_tokenizer_type": "sentencepiece",
     "mecab_kwargs": {

        "__type":"AddedToken"
     },
     "tokenize_chinese_chars":false,
+    "tokenizer_class": "DistilBertJapaneseTokenizer",
     "word_tokenizer_type": "mecab",
     "subword_tokenizer_type": "sentencepiece",
     "mecab_kwargs": {