| { | |
| "add_bos_token": true, | |
| "add_eos_token": false, | |
| "add_prefix_space": true, | |
| "added_tokens_decoder": { | |
| "0": { | |
| "content": "<unk>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "1": { | |
| "content": "<s>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "2": { | |
| "content": "</s>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": false | |
| }, | |
| "32000": { | |
| "content": "<|endoftext|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32001": { | |
| "content": "<|assistant|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32002": { | |
| "content": "<|step|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32003": { | |
| "content": "<|function_output|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32004": { | |
| "content": "<|tag|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32005": { | |
| "content": "<|function_call|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32006": { | |
| "content": "<|system|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32007": { | |
| "content": "<|end|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32008": { | |
| "content": "<|raw|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32009": { | |
| "content": "<|continue|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32010": { | |
| "content": "<|user|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32011": { | |
| "content": "<|function_list|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32012": { | |
| "content": "<|calc|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32013": { | |
| "content": "<|code|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32014": { | |
| "content": "<|/code|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32015": { | |
| "content": "<|summary|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32016": { | |
| "content": "<|resource|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32017": { | |
| "content": "<|assistant_mask|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32018": { | |
| "content": "<|start|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32019": { | |
| "content": "<|message|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32020": { | |
| "content": "<|fim_prefix|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32021": { | |
| "content": "<|fim_middle|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32022": { | |
| "content": "<|fim_suffix|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32023": { | |
| "content": "<|meta_start|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32024": { | |
| "content": "<|ipynb_marker|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32025": { | |
| "content": "<|diff_marker|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32026": { | |
| "content": "<|ghissue|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32027": { | |
| "content": "<|ghreview|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32028": { | |
| "content": "<|disc_start|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32029": { | |
| "content": "<|disc_sep|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32030": { | |
| "content": "<|disc_thread|><|query|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32031": { | |
| "content": "<|/query|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32032": { | |
| "content": "<|data|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32033": { | |
| "content": "<|/data|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32034": { | |
| "content": "<|sys|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32035": { | |
| "content": "<|/sys|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32036": { | |
| "content": "<|inst|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32037": { | |
| "content": "<|/inst|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": true, | |
| "single_word": false, | |
| "special": true | |
| } | |
| }, | |
| "additional_special_tokens": [ | |
| "<|/inst|>" | |
| ], | |
| "bos_token": "<s>", | |
| "chat_template": "{% if messages[0]['role'] == 'system' %}{{ '<|system|>' + messages[0]['content'] + '<|end|>\n' }}{% set loop_messages = messages[1:] %}{% else %}{% set loop_messages = messages %}{% endif %}{% for message in loop_messages %}{% if message['role'] == 'user' %}{{ '<|user|>' + message['content'] + '<|end|>\n' }}{% elif message['role'] == 'assistant' %}{{ '<|assistant|>' + message['content'] + '<|end|>\n' }}{% else %}{{ raise_exception('Only system, user, and assistant roles are supported!') }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% endif %}", | |
| "clean_up_tokenization_spaces": false, | |
| "eos_token": "<|end|>", | |
| "legacy": false, | |
| "model_max_length": 4096, | |
| "pad_token": "<|endoftext|>", | |
| "padding_side": "left", | |
| "sp_model_kwargs": {}, | |
| "spaces_between_special_tokens": false, | |
| "tokenizer_class": "LlamaTokenizer", | |
| "unk_token": "<unk>", | |
| "use_default_system_prompt": false | |
| } | |