nagrajn commited on
Commit
3e4395b
1 Parent(s): 9f9a2a3

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -27
  2. tokenizer_config.json +3 -2
tokenizer.json CHANGED
@@ -155,12 +155,6 @@
155
  "post_processor": {
156
  "type": "TemplateProcessing",
157
  "single": [
158
- {
159
- "SpecialToken": {
160
- "id": "<s>",
161
- "type_id": 0
162
- }
163
- },
164
  {
165
  "Sequence": {
166
  "id": "A",
@@ -175,12 +169,6 @@
175
  }
176
  ],
177
  "pair": [
178
- {
179
- "SpecialToken": {
180
- "id": "<s>",
181
- "type_id": 0
182
- }
183
- },
184
  {
185
  "Sequence": {
186
  "id": "A",
@@ -193,12 +181,6 @@
193
  "type_id": 0
194
  }
195
  },
196
- {
197
- "SpecialToken": {
198
- "id": "<s>",
199
- "type_id": 1
200
- }
201
- },
202
  {
203
  "Sequence": {
204
  "id": "B",
@@ -213,15 +195,6 @@
213
  }
214
  ],
215
  "special_tokens": {
216
- "<s>": {
217
- "id": "<s>",
218
- "ids": [
219
- 1
220
- ],
221
- "tokens": [
222
- "<s>"
223
- ]
224
- },
225
  "<|endoftext|>": {
226
  "id": "<|endoftext|>",
227
  "ids": [
 
155
  "post_processor": {
156
  "type": "TemplateProcessing",
157
  "single": [
 
 
 
 
 
 
158
  {
159
  "Sequence": {
160
  "id": "A",
 
169
  }
170
  ],
171
  "pair": [
 
 
 
 
 
 
172
  {
173
  "Sequence": {
174
  "id": "A",
 
181
  "type_id": 0
182
  }
183
  },
 
 
 
 
 
 
184
  {
185
  "Sequence": {
186
  "id": "B",
 
195
  }
196
  ],
197
  "special_tokens": {
 
 
 
 
 
 
 
 
 
198
  "<|endoftext|>": {
199
  "id": "<|endoftext|>",
200
  "ids": [
tokenizer_config.json CHANGED
@@ -1,6 +1,7 @@
1
  {
2
- "add_bos_token": true,
3
  "add_eos_token": true,
 
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
@@ -116,7 +117,7 @@
116
  }
117
  },
118
  "bos_token": "<s>",
119
- "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
120
  "clean_up_tokenization_spaces": false,
121
  "eos_token": "<|endoftext|>",
122
  "legacy": false,
 
1
  {
2
+ "add_bos_token": false,
3
  "add_eos_token": true,
4
+ "add_prefix_space": null,
5
  "added_tokens_decoder": {
6
  "0": {
7
  "content": "<unk>",
 
117
  }
118
  },
119
  "bos_token": "<s>",
120
+ "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
121
  "clean_up_tokenization_spaces": false,
122
  "eos_token": "<|endoftext|>",
123
  "legacy": false,