yzhangcs commited on
Commit
2c8b93b
1 Parent(s): 023f4e2

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -2
tokenizer_config.json CHANGED
@@ -30,12 +30,11 @@
30
  },
31
  "additional_special_tokens": [],
32
  "bos_token": "<s>",
33
- "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content }}{% endif %}{% endfor %}",
34
  "clean_up_tokenization_spaces": false,
35
  "eos_token": "</s>",
36
  "legacy": true,
37
  "model_max_length": 1000000000000000019884624838656,
38
- "pad_token": "</s>",
39
  "sp_model_kwargs": {},
40
  "spaces_between_special_tokens": false,
41
  "tokenizer_class": "LlamaTokenizer",
 
30
  },
31
  "additional_special_tokens": [],
32
  "bos_token": "<s>",
 
33
  "clean_up_tokenization_spaces": false,
34
  "eos_token": "</s>",
35
  "legacy": true,
36
  "model_max_length": 1000000000000000019884624838656,
37
+ "pad_token": null,
38
  "sp_model_kwargs": {},
39
  "spaces_between_special_tokens": false,
40
  "tokenizer_class": "LlamaTokenizer",