mychen76 commited on
Commit
6d956c0
1 Parent(s): 309315d

Upload tokenizer

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +7 -1
special_tokens_map.json CHANGED
@@ -21,5 +21,11 @@
21
  "rstrip": false,
22
  "single_word": false
23
  },
24
- "pad_token": "<|im_end|>"
 
 
 
 
 
 
25
  }
 
21
  "rstrip": false,
22
  "single_word": false
23
  },
24
+ "pad_token": {
25
+ "content": "<|im_end|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ }
31
  }