{ "unk_token": "<unk|LLM-jp>", "bos_token": "<s|LLM-jp>", "eos_token": "<EOD|LLM-jp>", "pad_token": "<pad|LLM-jp>", "cls_token": "<CLS|LLM-jp>", "sep_token": "<SEP|LLM-jp>", "eod_token": "<EOD|LLM-jp>", "mask_token": "<mask|LLM-jp>", "extra_ids": 0, "additional_special_tokens": [ "</s|LLM-jp>" ], "sp_model_kwargs": {}, "model_max_length": 1000000000000000019884624838656, "clean_up_tokenization_spaces": false, "special_tokens_map_file": null, "tokenizer_class": "PreTrainedTokenizerFast" }