zhaospei commited on
Commit
a9f7995
·
verified ·
1 Parent(s): e9c4553

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -3
tokenizer_config.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "add_bos_token": true,
3
  "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "32000": {
@@ -176,12 +176,12 @@
176
  "normalized": true,
177
  "rstrip": false,
178
  "single_word": false,
179
- "special": false
180
  }
181
  },
182
  "bos_token": "<|begin▁of▁sentence|>",
183
  "clean_up_tokenization_spaces": false,
184
- "eos_token": "<|end▁of▁sentence|>",
185
  "legacy": true,
186
  "model_max_length": 16384,
187
  "pad_token": "<|end▁of▁sentence|>",
 
1
  {
2
+ "add_bos_token": false,
3
  "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "32000": {
 
176
  "normalized": true,
177
  "rstrip": false,
178
  "single_word": false,
179
+ "special": true
180
  }
181
  },
182
  "bos_token": "<|begin▁of▁sentence|>",
183
  "clean_up_tokenization_spaces": false,
184
+ "eos_token": "<|EOT|>",
185
  "legacy": true,
186
  "model_max_length": 16384,
187
  "pad_token": "<|end▁of▁sentence|>",