File size: 162 Bytes
f9ae566
004c3d7
18d8f47
 
 
 
f9ae566
1
2
3
4
5
6
7
8
{
  "clean_up_tokenization_spaces": true,  
  "tokenizer_class": "SentencePieceJA",
  "auto_map": {
    "AutoTokenizer": "sentencepiece_ja.SentencePieceJA"
  }
}