|
{ |
|
"version": "1.0", |
|
"truncation": null, |
|
"padding": null, |
|
"added_tokens": [], |
|
"normalizer": null, |
|
"pre_tokenizer": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": false, |
|
"trim_offsets": true, |
|
"use_regex": true |
|
}, |
|
"post_processor": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": true, |
|
"trim_offsets": false, |
|
"use_regex": true |
|
}, |
|
"decoder": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": true, |
|
"trim_offsets": true, |
|
"use_regex": true |
|
}, |
|
"model": { |
|
"type": "BPE", |
|
"dropout": null, |
|
"unk_token": null, |
|
"continuing_subword_prefix": "", |
|
"end_of_word_suffix": "", |
|
"fuse_unk": false, |
|
"byte_fallback": false, |
|
"vocab": { |
|
"0": 0, |
|
"1": 1, |
|
"2": 2, |
|
"3": 3, |
|
"4": 4, |
|
"5": 5, |
|
"6": 6, |
|
"7": 7, |
|
"8": 8, |
|
"9": 9, |
|
"<|endoftext|>": 10 |
|
}, |
|
"merges": [] |
|
} |
|
} |