Emm9625 commited on
Commit
85b3edf
·
verified ·
1 Parent(s): d2186df

Upload 3 files

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +2 -1
  2. tokenizer_config.json +1 -1
special_tokens_map.json CHANGED
@@ -1,4 +1,5 @@
1
  {
2
  "bos_token": "<|begin_of_text|>",
3
- "eos_token": "<|im_end|>"
 
4
  }
 
1
  {
2
  "bos_token": "<|begin_of_text|>",
3
+ "eos_token": "<|im_end|>",
4
+ "pad_token": "<|finetune_right_pad_id|>"
5
  }
tokenizer_config.json CHANGED
@@ -2055,7 +2055,7 @@
2055
  "eos_token": "<|im_end|>",
2056
  "extra_special_tokens": {},
2057
  "model_max_length": 1000000000000000019884624838656,
2058
- "pad_token": null,
2059
  "tokenizer_class": "PreTrainedTokenizerFast",
2060
  "unk_token": null
2061
  }
 
2055
  "eos_token": "<|im_end|>",
2056
  "extra_special_tokens": {},
2057
  "model_max_length": 1000000000000000019884624838656,
2058
+ "pad_token": "<|finetune_right_pad_id|>",
2059
  "tokenizer_class": "PreTrainedTokenizerFast",
2060
  "unk_token": null
2061
  }