{ "add_bos_token": true, "add_eos_token": false, "add_prefix_space": false, "added_tokens_decoder": { "0": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "3": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "4": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "5": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "6": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "7": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "255000": { "content": "<|START_OF_TURN_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255001": { "content": "<|END_OF_TURN_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "255002": { "content": "<|YES_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255003": { "content": "<|NO_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255004": { "content": "<|GOOD_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255005": { "content": "<|BAD_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255006": { "content": "<|USER_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255007": { "content": "<|CHATBOT_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255008": { "content": "<|SYSTEM_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255009": { "content": "<|USER_0_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255010": { "content": "<|USER_1_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255011": { "content": "<|USER_2_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255012": { "content": "<|USER_3_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255013": { "content": "<|USER_4_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255014": { "content": "<|USER_5_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255015": { "content": "<|USER_6_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255016": { "content": "<|USER_7_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255017": { "content": "<|USER_8_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255018": { "content": "<|USER_9_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255019": { "content": "<|EXTRA_0_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255020": { "content": "<|EXTRA_1_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255021": { "content": "<|EXTRA_2_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255022": { "content": "<|EXTRA_3_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255023": { "content": "<|EXTRA_4_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255024": { "content": "<|EXTRA_5_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255025": { "content": "<|EXTRA_6_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255026": { "content": "<|EXTRA_7_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255027": { "content": "<|EXTRA_8_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "255028": { "content": "<|EXTRA_9_TOKEN|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false } }, "bos_token": "", "chat_template": [ { "name": "default", "template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif false == true %}{% set loop_messages = messages %}{% set system_message = 'You are Aya, a brilliant, sophisticated, multilingual AI-assistant trained to assist human users by providing thorough responses. You are able to interact and respond to questions in 23 languages and you are powered by a multilingual model built by Cohere For AI.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% if system_message != false %}{{ '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' + system_message + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% elif message['role'] == 'assistant' %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}{% endif %}" } ], "clean_up_tokenization_spaces": false, "eos_token": "<|END_OF_TURN_TOKEN|>", "extra_special_tokens": {}, "legacy": true, "merges_file": null, "model_max_length": 1000000000000000019884624838656, "pad_token": "", "sp_model_kwargs": {}, "spaces_between_special_tokens": false, "tokenizer_class": "CohereTokenizer", "unk_token": null, "use_default_system_prompt": false, "vocab_file": null }