SF-Foundation commited on
Commit
419d7ea
·
verified ·
1 Parent(s): 84aff42

Upload special_tokens_map.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +531 -0
special_tokens_map.json ADDED
@@ -0,0 +1,531 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "[control_1]",
4
+ "[control_2]",
5
+ "[control_3]",
6
+ "[control_4]",
7
+ "[control_5]",
8
+ "[control_6]",
9
+ "[control_7]",
10
+ "[control_8]",
11
+ "[control_9]",
12
+ "[control_10]",
13
+ "[control_11]",
14
+ "[control_12]",
15
+ "[control_13]",
16
+ "[control_14]",
17
+ "[control_15]",
18
+ "[control_16]",
19
+ "[control_17]",
20
+ "[control_18]",
21
+ "[control_19]",
22
+ "[control_20]",
23
+ "[control_21]",
24
+ "[control_22]",
25
+ "[control_23]",
26
+ "[control_24]",
27
+ "[control_25]",
28
+ "[control_26]",
29
+ "[control_27]",
30
+ "[control_28]",
31
+ "[control_29]",
32
+ "[control_30]",
33
+ "[control_31]",
34
+ "[control_32]",
35
+ "[control_33]",
36
+ "[control_34]",
37
+ "[control_35]",
38
+ "[control_36]",
39
+ "[control_37]",
40
+ "[control_38]",
41
+ "[control_39]",
42
+ "[control_40]",
43
+ "[control_41]",
44
+ "[control_42]",
45
+ "[control_43]",
46
+ "[control_44]",
47
+ "[control_45]",
48
+ "[control_46]",
49
+ "[control_47]",
50
+ "[control_48]",
51
+ "[control_49]",
52
+ "[control_50]",
53
+ "[control_51]",
54
+ "[control_52]",
55
+ "[control_53]",
56
+ "[control_54]",
57
+ "[control_55]",
58
+ "[control_56]",
59
+ "[control_57]",
60
+ "[control_58]",
61
+ "[control_59]",
62
+ "[control_60]",
63
+ "[control_61]",
64
+ "[control_62]",
65
+ "[control_63]",
66
+ "[control_64]",
67
+ "[control_65]",
68
+ "[control_66]",
69
+ "[control_67]",
70
+ "[control_68]",
71
+ "[control_69]",
72
+ "[control_70]",
73
+ "[control_71]",
74
+ "[control_72]",
75
+ "[control_73]",
76
+ "[control_74]",
77
+ "[control_75]",
78
+ "[control_76]",
79
+ "[control_77]",
80
+ "[control_78]",
81
+ "[control_79]",
82
+ "[control_80]",
83
+ "[control_81]",
84
+ "[control_82]",
85
+ "[control_83]",
86
+ "[control_84]",
87
+ "[control_85]",
88
+ "[control_86]",
89
+ "[control_87]",
90
+ "[control_88]",
91
+ "[control_89]",
92
+ "[control_90]",
93
+ "[control_91]",
94
+ "[control_92]",
95
+ "[control_93]",
96
+ "[control_94]",
97
+ "[control_95]",
98
+ "[control_96]",
99
+ "[control_97]",
100
+ "[control_98]",
101
+ "[control_99]",
102
+ "[control_100]",
103
+ "[control_101]",
104
+ "[control_102]",
105
+ "[control_103]",
106
+ "[control_104]",
107
+ "[control_105]",
108
+ "[control_106]",
109
+ "[control_107]",
110
+ "[control_108]",
111
+ "[control_109]",
112
+ "[control_110]",
113
+ "[control_111]",
114
+ "[control_112]",
115
+ "[control_113]",
116
+ "[control_114]",
117
+ "[control_115]",
118
+ "[control_116]",
119
+ "[control_117]",
120
+ "[control_118]",
121
+ "[control_119]",
122
+ "[control_120]",
123
+ "[control_121]",
124
+ "[control_122]",
125
+ "[control_123]",
126
+ "[control_124]",
127
+ "[control_125]",
128
+ "[control_126]",
129
+ "[control_127]",
130
+ "[control_128]",
131
+ "[control_129]",
132
+ "[control_130]",
133
+ "[control_131]",
134
+ "[control_132]",
135
+ "[control_133]",
136
+ "[control_134]",
137
+ "[control_135]",
138
+ "[control_136]",
139
+ "[control_137]",
140
+ "[control_138]",
141
+ "[control_139]",
142
+ "[control_140]",
143
+ "[control_141]",
144
+ "[control_142]",
145
+ "[control_143]",
146
+ "[control_144]",
147
+ "[control_145]",
148
+ "[control_146]",
149
+ "[control_147]",
150
+ "[control_148]",
151
+ "[control_149]",
152
+ "[control_150]",
153
+ "[control_151]",
154
+ "[control_152]",
155
+ "[control_153]",
156
+ "[control_154]",
157
+ "[control_155]",
158
+ "[control_156]",
159
+ "[control_157]",
160
+ "[control_158]",
161
+ "[control_159]",
162
+ "[control_160]",
163
+ "[control_161]",
164
+ "[control_162]",
165
+ "[control_163]",
166
+ "[control_164]",
167
+ "[control_165]",
168
+ "[control_166]",
169
+ "[control_167]",
170
+ "[control_168]",
171
+ "[control_169]",
172
+ "[control_170]",
173
+ "[control_171]",
174
+ "[control_172]",
175
+ "[control_173]",
176
+ "[control_174]",
177
+ "[control_175]",
178
+ "[control_176]",
179
+ "[control_177]",
180
+ "[control_178]",
181
+ "[control_179]",
182
+ "[control_180]",
183
+ "[control_181]",
184
+ "[control_182]",
185
+ "[control_183]",
186
+ "[control_184]",
187
+ "[control_185]",
188
+ "[control_186]",
189
+ "[control_187]",
190
+ "[control_188]",
191
+ "[control_189]",
192
+ "[control_190]",
193
+ "[control_191]",
194
+ "[control_192]",
195
+ "[control_193]",
196
+ "[control_194]",
197
+ "[control_195]",
198
+ "[control_196]",
199
+ "[control_197]",
200
+ "[control_198]",
201
+ "[control_199]",
202
+ "[control_200]",
203
+ "[control_201]",
204
+ "[control_202]",
205
+ "[control_203]",
206
+ "[control_204]",
207
+ "[control_205]",
208
+ "[control_206]",
209
+ "[control_207]",
210
+ "[control_208]",
211
+ "[control_209]",
212
+ "[control_210]",
213
+ "[control_211]",
214
+ "[control_212]",
215
+ "[control_213]",
216
+ "[control_214]",
217
+ "[control_215]",
218
+ "[control_216]",
219
+ "[control_217]",
220
+ "[control_218]",
221
+ "[control_219]",
222
+ "[control_220]",
223
+ "[control_221]",
224
+ "[control_222]",
225
+ "[control_223]",
226
+ "[control_224]",
227
+ "[control_225]",
228
+ "[control_226]",
229
+ "[control_227]",
230
+ "[control_228]",
231
+ "[control_229]",
232
+ "[control_230]",
233
+ "[control_231]",
234
+ "[control_232]",
235
+ "[control_233]",
236
+ "[control_234]",
237
+ "[control_235]",
238
+ "[control_236]",
239
+ "[control_237]",
240
+ "[control_238]",
241
+ "[control_239]",
242
+ "[control_240]",
243
+ "[control_241]",
244
+ "[control_242]",
245
+ "[control_243]",
246
+ "[control_244]",
247
+ "[control_245]",
248
+ "[control_246]",
249
+ "[control_247]",
250
+ "[control_248]",
251
+ "[control_249]",
252
+ "[control_250]",
253
+ "[control_251]",
254
+ "[control_252]",
255
+ "[control_253]",
256
+ "[control_254]",
257
+ "[control_255]",
258
+ "[control_256]",
259
+ "[control_257]",
260
+ "[control_258]",
261
+ "[control_259]",
262
+ "[control_260]",
263
+ "[control_261]",
264
+ "[control_262]",
265
+ "[control_263]",
266
+ "[control_264]",
267
+ "[control_265]",
268
+ "[control_266]",
269
+ "[control_267]",
270
+ "[control_268]",
271
+ "[control_269]",
272
+ "[control_270]",
273
+ "[control_271]",
274
+ "[control_272]",
275
+ "[control_273]",
276
+ "[control_274]",
277
+ "[control_275]",
278
+ "[control_276]",
279
+ "[control_277]",
280
+ "[control_278]",
281
+ "[control_279]",
282
+ "[control_280]",
283
+ "[control_281]",
284
+ "[control_282]",
285
+ "[control_283]",
286
+ "[control_284]",
287
+ "[control_285]",
288
+ "[control_286]",
289
+ "[control_287]",
290
+ "[control_288]",
291
+ "[control_289]",
292
+ "[control_290]",
293
+ "[control_291]",
294
+ "[control_292]",
295
+ "[control_293]",
296
+ "[control_294]",
297
+ "[control_295]",
298
+ "[control_296]",
299
+ "[control_297]",
300
+ "[control_298]",
301
+ "[control_299]",
302
+ "[control_300]",
303
+ "[control_301]",
304
+ "[control_302]",
305
+ "[control_303]",
306
+ "[control_304]",
307
+ "[control_305]",
308
+ "[control_306]",
309
+ "[control_307]",
310
+ "[control_308]",
311
+ "[control_309]",
312
+ "[control_310]",
313
+ "[control_311]",
314
+ "[control_312]",
315
+ "[control_313]",
316
+ "[control_314]",
317
+ "[control_315]",
318
+ "[control_316]",
319
+ "[control_317]",
320
+ "[control_318]",
321
+ "[control_319]",
322
+ "[control_320]",
323
+ "[control_321]",
324
+ "[control_322]",
325
+ "[control_323]",
326
+ "[control_324]",
327
+ "[control_325]",
328
+ "[control_326]",
329
+ "[control_327]",
330
+ "[control_328]",
331
+ "[control_329]",
332
+ "[control_330]",
333
+ "[control_331]",
334
+ "[control_332]",
335
+ "[control_333]",
336
+ "[control_334]",
337
+ "[control_335]",
338
+ "[control_336]",
339
+ "[control_337]",
340
+ "[control_338]",
341
+ "[control_339]",
342
+ "[control_340]",
343
+ "[control_341]",
344
+ "[control_342]",
345
+ "[control_343]",
346
+ "[control_344]",
347
+ "[control_345]",
348
+ "[control_346]",
349
+ "[control_347]",
350
+ "[control_348]",
351
+ "[control_349]",
352
+ "[control_350]",
353
+ "[control_351]",
354
+ "[control_352]",
355
+ "[control_353]",
356
+ "[control_354]",
357
+ "[control_355]",
358
+ "[control_356]",
359
+ "[control_357]",
360
+ "[control_358]",
361
+ "[control_359]",
362
+ "[control_360]",
363
+ "[control_361]",
364
+ "[control_362]",
365
+ "[control_363]",
366
+ "[control_364]",
367
+ "[control_365]",
368
+ "[control_366]",
369
+ "[control_367]",
370
+ "[control_368]",
371
+ "[control_369]",
372
+ "[control_370]",
373
+ "[control_371]",
374
+ "[control_372]",
375
+ "[control_373]",
376
+ "[control_374]",
377
+ "[control_375]",
378
+ "[control_376]",
379
+ "[control_377]",
380
+ "[control_378]",
381
+ "[control_379]",
382
+ "[control_380]",
383
+ "[control_381]",
384
+ "[control_382]",
385
+ "[control_383]",
386
+ "[control_384]",
387
+ "[control_385]",
388
+ "[control_386]",
389
+ "[control_387]",
390
+ "[control_388]",
391
+ "[control_389]",
392
+ "[control_390]",
393
+ "[control_391]",
394
+ "[control_392]",
395
+ "[control_393]",
396
+ "[control_394]",
397
+ "[control_395]",
398
+ "[control_396]",
399
+ "[control_397]",
400
+ "[control_398]",
401
+ "[control_399]",
402
+ "[control_400]",
403
+ "[control_401]",
404
+ "[control_402]",
405
+ "[control_403]",
406
+ "[control_404]",
407
+ "[control_405]",
408
+ "[control_406]",
409
+ "[control_407]",
410
+ "[control_408]",
411
+ "[control_409]",
412
+ "[control_410]",
413
+ "[control_411]",
414
+ "[control_412]",
415
+ "[control_413]",
416
+ "[control_414]",
417
+ "[control_415]",
418
+ "[control_416]",
419
+ "[control_417]",
420
+ "[control_418]",
421
+ "[control_419]",
422
+ "[control_420]",
423
+ "[control_421]",
424
+ "[control_422]",
425
+ "[control_423]",
426
+ "[control_424]",
427
+ "[control_425]",
428
+ "[control_426]",
429
+ "[control_427]",
430
+ "[control_428]",
431
+ "[control_429]",
432
+ "[control_430]",
433
+ "[control_431]",
434
+ "[control_432]",
435
+ "[control_433]",
436
+ "[control_434]",
437
+ "[control_435]",
438
+ "[control_436]",
439
+ "[control_437]",
440
+ "[control_438]",
441
+ "[control_439]",
442
+ "[control_440]",
443
+ "[control_441]",
444
+ "[control_442]",
445
+ "[control_443]",
446
+ "[control_444]",
447
+ "[control_445]",
448
+ "[control_446]",
449
+ "[control_447]",
450
+ "[control_448]",
451
+ "[control_449]",
452
+ "[control_450]",
453
+ "[control_451]",
454
+ "[control_452]",
455
+ "[control_453]",
456
+ "[control_454]",
457
+ "[control_455]",
458
+ "[control_456]",
459
+ "[control_457]",
460
+ "[control_458]",
461
+ "[control_459]",
462
+ "[control_460]",
463
+ "[control_461]",
464
+ "[control_462]",
465
+ "[control_463]",
466
+ "[control_464]",
467
+ "[control_465]",
468
+ "[control_466]",
469
+ "[control_467]",
470
+ "[control_468]",
471
+ "[control_469]",
472
+ "[control_470]",
473
+ "[control_471]",
474
+ "[control_472]",
475
+ "[control_473]",
476
+ "[control_474]",
477
+ "[control_475]",
478
+ "[control_476]",
479
+ "[control_477]",
480
+ "[control_478]",
481
+ "[control_479]",
482
+ "[control_480]",
483
+ "[control_481]",
484
+ "[control_482]",
485
+ "[control_483]",
486
+ "[control_484]",
487
+ "[control_485]",
488
+ "[control_486]",
489
+ "[control_487]",
490
+ "[control_488]",
491
+ "[control_489]",
492
+ "[control_490]",
493
+ "[control_491]",
494
+ "[control_492]",
495
+ "[control_493]",
496
+ "[control_494]",
497
+ "[control_495]",
498
+ "[control_496]",
499
+ "[control_497]",
500
+ "[control_498]",
501
+ "[control_499]"
502
+ ],
503
+ "bos_token": {
504
+ "content": "<s>",
505
+ "lstrip": false,
506
+ "normalized": false,
507
+ "rstrip": false,
508
+ "single_word": false
509
+ },
510
+ "eos_token": {
511
+ "content": "</s>",
512
+ "lstrip": false,
513
+ "normalized": false,
514
+ "rstrip": false,
515
+ "single_word": false
516
+ },
517
+ "pad_token": {
518
+ "content": "<pad>",
519
+ "lstrip": false,
520
+ "normalized": false,
521
+ "rstrip": false,
522
+ "single_word": false
523
+ },
524
+ "unk_token": {
525
+ "content": "<unk>",
526
+ "lstrip": false,
527
+ "normalized": false,
528
+ "rstrip": false,
529
+ "single_word": false
530
+ }
531
+ }