Undi95 commited on
Commit
9ecf4a7
1 Parent(s): acfc2c6

Update added_tokens.json

Browse files
Files changed (1) hide show
  1. added_tokens.json +5 -2
added_tokens.json CHANGED
@@ -1,5 +1,8 @@
1
  {
2
  "</s>": 2,
3
  "<s>": 1,
4
- "<unk>": 0
5
- }
 
 
 
 
1
  {
2
  "</s>": 2,
3
  "<s>": 1,
4
+ "<unk>": 0,
5
+ "<|im_end|>": 32000,
6
+ "<|im_start|>": 32001,
7
+ "[PAD]": 32002
8
+ }