f9ae566 004c3d7 18d8f47 f9ae566
1
2
3
4
5
6
7
8
{ "clean_up_tokenization_spaces": true, "tokenizer_class": "SentencePieceJA", "auto_map": { "AutoTokenizer": "sentencepiece_ja.SentencePieceJA" } }