Text Generation
Transformers
PyTorch
Indonesian
English
mistral
conversational
text-generation-inference
Inference Endpoints
Ichsan2895 commited on
Commit
0a95b34
1 Parent(s): bf9c193

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -5
tokenizer_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "add_bos_token": true,
3
- "add_eos_token": true,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
@@ -53,15 +53,13 @@
53
  "bos_token": "<s>",
54
  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
55
  "clean_up_tokenization_spaces": false,
56
- "eos_token": "<|im_end|>",
57
  "legacy": true,
58
  "model_max_length": 1000000000000000019884624838656,
59
  "pad_token": "</s>",
60
  "sp_model_kwargs": {},
61
  "spaces_between_special_tokens": false,
62
  "tokenizer_class": "LlamaTokenizer",
63
- "trust_remote_code": false,
64
  "unk_token": "<unk>",
65
- "use_default_system_prompt": true,
66
- "use_fast": true
67
  }
 
1
  {
2
  "add_bos_token": true,
3
+ "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
 
53
  "bos_token": "<s>",
54
  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
55
  "clean_up_tokenization_spaces": false,
56
+ "eos_token": "</s>",
57
  "legacy": true,
58
  "model_max_length": 1000000000000000019884624838656,
59
  "pad_token": "</s>",
60
  "sp_model_kwargs": {},
61
  "spaces_between_special_tokens": false,
62
  "tokenizer_class": "LlamaTokenizer",
 
63
  "unk_token": "<unk>",
64
+ "use_default_system_prompt": true
 
65
  }