zhiyuan8 commited on
Commit
887166b
·
verified ·
1 Parent(s): d324e84

Update special_tokens_map.json

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +6 -5
special_tokens_map.json CHANGED
@@ -1,5 +1,6 @@
1
- {
2
- "bos_token": "<|rwkv_tokenizer_end_of_text|>",
3
- "eos_token": "<|rwkv_tokenizer_end_of_text|>",
4
- "unk_token": "<|rwkv_tokenizer_end_of_text|>"
5
- }
 
 
1
+ {
2
+ "bos_token": "<|rwkv_tokenizer_end_of_text|>",
3
+ "eos_token": "\n\n",
4
+ "unk_token": "<|rwkv_tokenizer_end_of_text|>",
5
+ "pad_token": "<|rwkv_tokenizer_end_of_text|>"
6
+ }