pengyunie commited on
Commit
d8b90ce
·
verified ·
1 Parent(s): 9ea6639

Upload tokenizer

Browse files
added_tokens.json ADDED
@@ -0,0 +1,218 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ " <C>": 32100,
3
+ " <M>": 32101,
4
+ " <decl>": 32102,
5
+ " <gF>": 32103,
6
+ " <gL>": 32104,
7
+ " <multi>": 32105,
8
+ " <pF>": 32106,
9
+ " <pL>": 32107,
10
+ " <ret>": 32108,
11
+ " <throw>": 32109,
12
+ " AALOAD": 32110,
13
+ " AASTORE": 32111,
14
+ " ACONST_NULL": 32112,
15
+ " ALOAD": 32113,
16
+ " ALOAD_0": 32114,
17
+ " ALOAD_1": 32115,
18
+ " ALOAD_2": 32116,
19
+ " ALOAD_3": 32117,
20
+ " ANEWARRAY": 32118,
21
+ " ARETURN": 32119,
22
+ " ARRAYLENGTH": 32120,
23
+ " ASTORE": 32121,
24
+ " ASTORE_0": 32122,
25
+ " ASTORE_1": 32123,
26
+ " ASTORE_2": 32124,
27
+ " ASTORE_3": 32125,
28
+ " ATHROW": 32126,
29
+ " BALOAD": 32127,
30
+ " BASTORE": 32128,
31
+ " BIPUSH": 32129,
32
+ " CALOAD": 32130,
33
+ " CASTORE": 32131,
34
+ " CHECKCAST": 32132,
35
+ " D2F": 32133,
36
+ " D2I": 32134,
37
+ " D2L": 32135,
38
+ " DADD": 32136,
39
+ " DALOAD": 32137,
40
+ " DASTORE": 32138,
41
+ " DCMPG": 32139,
42
+ " DCMPL": 32140,
43
+ " DCONST_0": 32141,
44
+ " DCONST_1": 32142,
45
+ " DDIV": 32143,
46
+ " DLOAD": 32144,
47
+ " DLOAD_0": 32145,
48
+ " DLOAD_1": 32146,
49
+ " DLOAD_2": 32147,
50
+ " DLOAD_3": 32148,
51
+ " DMUL": 32149,
52
+ " DNEG": 32150,
53
+ " DREM": 32151,
54
+ " DRETURN": 32152,
55
+ " DSTORE": 32153,
56
+ " DSTORE_0": 32154,
57
+ " DSTORE_1": 32155,
58
+ " DSTORE_2": 32156,
59
+ " DSTORE_3": 32157,
60
+ " DSUB": 32158,
61
+ " DUP": 32159,
62
+ " DUP2": 32160,
63
+ " DUP2_X1": 32161,
64
+ " DUP2_X2": 32162,
65
+ " DUP_X1": 32163,
66
+ " DUP_X2": 32164,
67
+ " F2D": 32165,
68
+ " F2I": 32166,
69
+ " F2L": 32167,
70
+ " FADD": 32168,
71
+ " FALOAD": 32169,
72
+ " FASTORE": 32170,
73
+ " FCMPG": 32171,
74
+ " FCMPL": 32172,
75
+ " FCONST_0": 32173,
76
+ " FCONST_1": 32174,
77
+ " FCONST_2": 32175,
78
+ " FDIV": 32176,
79
+ " FLOAD": 32177,
80
+ " FLOAD_0": 32178,
81
+ " FLOAD_1": 32179,
82
+ " FLOAD_2": 32180,
83
+ " FLOAD_3": 32181,
84
+ " FMUL": 32182,
85
+ " FNEG": 32183,
86
+ " FREM": 32184,
87
+ " FRETURN": 32185,
88
+ " FSTORE": 32186,
89
+ " FSTORE_0": 32187,
90
+ " FSTORE_1": 32188,
91
+ " FSTORE_2": 32189,
92
+ " FSTORE_3": 32190,
93
+ " FSUB": 32191,
94
+ " GETFIELD": 32192,
95
+ " GETSTATIC": 32193,
96
+ " GOTO": 32194,
97
+ " GOTO_W": 32195,
98
+ " I2B": 32196,
99
+ " I2C": 32197,
100
+ " I2D": 32198,
101
+ " I2F": 32199,
102
+ " I2L": 32200,
103
+ " I2S": 32201,
104
+ " IADD": 32202,
105
+ " IALOAD": 32203,
106
+ " IAND": 32204,
107
+ " IASTORE": 32205,
108
+ " ICONST_0": 32206,
109
+ " ICONST_1": 32207,
110
+ " ICONST_2": 32208,
111
+ " ICONST_3": 32209,
112
+ " ICONST_4": 32210,
113
+ " ICONST_5": 32211,
114
+ " ICONST_M1": 32212,
115
+ " IDIV": 32213,
116
+ " IFEQ": 32214,
117
+ " IFGE": 32215,
118
+ " IFGT": 32216,
119
+ " IFLE": 32217,
120
+ " IFLT": 32218,
121
+ " IFNE": 32219,
122
+ " IFNONNULL": 32220,
123
+ " IFNULL": 32221,
124
+ " IF_ACMPEQ": 32222,
125
+ " IF_ACMPNE": 32223,
126
+ " IF_ICMPEQ": 32224,
127
+ " IF_ICMPGE": 32225,
128
+ " IF_ICMPGT": 32226,
129
+ " IF_ICMPLE": 32227,
130
+ " IF_ICMPLT": 32228,
131
+ " IF_ICMPNE": 32229,
132
+ " IINC": 32230,
133
+ " ILOAD": 32231,
134
+ " ILOAD_0": 32232,
135
+ " ILOAD_1": 32233,
136
+ " ILOAD_2": 32234,
137
+ " ILOAD_3": 32235,
138
+ " IMUL": 32236,
139
+ " INEG": 32237,
140
+ " INSTANCEOF": 32238,
141
+ " INVOKEDYNAMIC": 32239,
142
+ " INVOKEINTERFACE": 32240,
143
+ " INVOKESPECIAL": 32241,
144
+ " INVOKESTATIC": 32242,
145
+ " INVOKEVIRTUAL": 32243,
146
+ " IOR": 32244,
147
+ " IREM": 32245,
148
+ " IRETURN": 32246,
149
+ " ISHL": 32247,
150
+ " ISHR": 32248,
151
+ " ISTORE": 32249,
152
+ " ISTORE_0": 32250,
153
+ " ISTORE_1": 32251,
154
+ " ISTORE_2": 32252,
155
+ " ISTORE_3": 32253,
156
+ " ISUB": 32254,
157
+ " IUSHR": 32255,
158
+ " IXOR": 32256,
159
+ " JSR": 32257,
160
+ " JSR_W": 32258,
161
+ " L2D": 32259,
162
+ " L2F": 32260,
163
+ " L2I": 32261,
164
+ " LADD": 32262,
165
+ " LALOAD": 32263,
166
+ " LAND": 32264,
167
+ " LASTORE": 32265,
168
+ " LCMP": 32266,
169
+ " LCONST_0": 32267,
170
+ " LCONST_1": 32268,
171
+ " LDC": 32269,
172
+ " LDC2_W": 32270,
173
+ " LDC_END": 32271,
174
+ " LDC_W": 32272,
175
+ " LDIV": 32273,
176
+ " LLOAD": 32274,
177
+ " LLOAD_0": 32275,
178
+ " LLOAD_1": 32276,
179
+ " LLOAD_2": 32277,
180
+ " LLOAD_3": 32278,
181
+ " LMUL": 32279,
182
+ " LNEG": 32280,
183
+ " LOOKUPSWITCH": 32281,
184
+ " LOR": 32282,
185
+ " LREM": 32283,
186
+ " LRETURN": 32284,
187
+ " LSHL": 32285,
188
+ " LSHR": 32286,
189
+ " LSTORE": 32287,
190
+ " LSTORE_0": 32288,
191
+ " LSTORE_1": 32289,
192
+ " LSTORE_2": 32290,
193
+ " LSTORE_3": 32291,
194
+ " LSUB": 32292,
195
+ " LUSHR": 32293,
196
+ " LXOR": 32294,
197
+ " MONITORENTER": 32295,
198
+ " MONITOREXIT": 32296,
199
+ " MULTIANEWARRAY": 32297,
200
+ " NEW": 32298,
201
+ " NEWARRAY": 32299,
202
+ " NOP": 32300,
203
+ " POP": 32301,
204
+ " POP2": 32302,
205
+ " PUTFIELD": 32303,
206
+ " PUTSTATIC": 32304,
207
+ " RET": 32305,
208
+ " RETURN": 32306,
209
+ " SALOAD": 32307,
210
+ " SASTORE": 32308,
211
+ " SIPUSH": 32309,
212
+ " SWAP": 32310,
213
+ " TABLESWITCH": 32311,
214
+ " WIDE": 32312,
215
+ "<cls>": 32314,
216
+ "<sep>": 32315,
217
+ "<|UNKNOWN|>": 32313
218
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,330 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ " <C>",
4
+ " <M>",
5
+ " <decl>",
6
+ " <gF>",
7
+ " <gL>",
8
+ " <multi>",
9
+ " <pF>",
10
+ " <pL>",
11
+ " <ret>",
12
+ " <throw>",
13
+ " AALOAD",
14
+ " AASTORE",
15
+ " ACONST_NULL",
16
+ " ALOAD",
17
+ " ALOAD_0",
18
+ " ALOAD_1",
19
+ " ALOAD_2",
20
+ " ALOAD_3",
21
+ " ANEWARRAY",
22
+ " ARETURN",
23
+ " ARRAYLENGTH",
24
+ " ASTORE",
25
+ " ASTORE_0",
26
+ " ASTORE_1",
27
+ " ASTORE_2",
28
+ " ASTORE_3",
29
+ " ATHROW",
30
+ " BALOAD",
31
+ " BASTORE",
32
+ " BIPUSH",
33
+ " CALOAD",
34
+ " CASTORE",
35
+ " CHECKCAST",
36
+ " D2F",
37
+ " D2I",
38
+ " D2L",
39
+ " DADD",
40
+ " DALOAD",
41
+ " DASTORE",
42
+ " DCMPG",
43
+ " DCMPL",
44
+ " DCONST_0",
45
+ " DCONST_1",
46
+ " DDIV",
47
+ " DLOAD",
48
+ " DLOAD_0",
49
+ " DLOAD_1",
50
+ " DLOAD_2",
51
+ " DLOAD_3",
52
+ " DMUL",
53
+ " DNEG",
54
+ " DREM",
55
+ " DRETURN",
56
+ " DSTORE",
57
+ " DSTORE_0",
58
+ " DSTORE_1",
59
+ " DSTORE_2",
60
+ " DSTORE_3",
61
+ " DSUB",
62
+ " DUP",
63
+ " DUP2",
64
+ " DUP2_X1",
65
+ " DUP2_X2",
66
+ " DUP_X1",
67
+ " DUP_X2",
68
+ " F2D",
69
+ " F2I",
70
+ " F2L",
71
+ " FADD",
72
+ " FALOAD",
73
+ " FASTORE",
74
+ " FCMPG",
75
+ " FCMPL",
76
+ " FCONST_0",
77
+ " FCONST_1",
78
+ " FCONST_2",
79
+ " FDIV",
80
+ " FLOAD",
81
+ " FLOAD_0",
82
+ " FLOAD_1",
83
+ " FLOAD_2",
84
+ " FLOAD_3",
85
+ " FMUL",
86
+ " FNEG",
87
+ " FREM",
88
+ " FRETURN",
89
+ " FSTORE",
90
+ " FSTORE_0",
91
+ " FSTORE_1",
92
+ " FSTORE_2",
93
+ " FSTORE_3",
94
+ " FSUB",
95
+ " GETFIELD",
96
+ " GETSTATIC",
97
+ " GOTO",
98
+ " GOTO_W",
99
+ " I2B",
100
+ " I2C",
101
+ " I2D",
102
+ " I2F",
103
+ " I2L",
104
+ " I2S",
105
+ " IADD",
106
+ " IALOAD",
107
+ " IAND",
108
+ " IASTORE",
109
+ " ICONST_0",
110
+ " ICONST_1",
111
+ " ICONST_2",
112
+ " ICONST_3",
113
+ " ICONST_4",
114
+ " ICONST_5",
115
+ " ICONST_M1",
116
+ " IDIV",
117
+ " IFEQ",
118
+ " IFGE",
119
+ " IFGT",
120
+ " IFLE",
121
+ " IFLT",
122
+ " IFNE",
123
+ " IFNONNULL",
124
+ " IFNULL",
125
+ " IF_ACMPEQ",
126
+ " IF_ACMPNE",
127
+ " IF_ICMPEQ",
128
+ " IF_ICMPGE",
129
+ " IF_ICMPGT",
130
+ " IF_ICMPLE",
131
+ " IF_ICMPLT",
132
+ " IF_ICMPNE",
133
+ " IINC",
134
+ " ILOAD",
135
+ " ILOAD_0",
136
+ " ILOAD_1",
137
+ " ILOAD_2",
138
+ " ILOAD_3",
139
+ " IMUL",
140
+ " INEG",
141
+ " INSTANCEOF",
142
+ " INVOKEDYNAMIC",
143
+ " INVOKEINTERFACE",
144
+ " INVOKESPECIAL",
145
+ " INVOKESTATIC",
146
+ " INVOKEVIRTUAL",
147
+ " IOR",
148
+ " IREM",
149
+ " IRETURN",
150
+ " ISHL",
151
+ " ISHR",
152
+ " ISTORE",
153
+ " ISTORE_0",
154
+ " ISTORE_1",
155
+ " ISTORE_2",
156
+ " ISTORE_3",
157
+ " ISUB",
158
+ " IUSHR",
159
+ " IXOR",
160
+ " JSR",
161
+ " JSR_W",
162
+ " L2D",
163
+ " L2F",
164
+ " L2I",
165
+ " LADD",
166
+ " LALOAD",
167
+ " LAND",
168
+ " LASTORE",
169
+ " LCMP",
170
+ " LCONST_0",
171
+ " LCONST_1",
172
+ " LDC",
173
+ " LDC2_W",
174
+ " LDC_END",
175
+ " LDC_W",
176
+ " LDIV",
177
+ " LLOAD",
178
+ " LLOAD_0",
179
+ " LLOAD_1",
180
+ " LLOAD_2",
181
+ " LLOAD_3",
182
+ " LMUL",
183
+ " LNEG",
184
+ " LOOKUPSWITCH",
185
+ " LOR",
186
+ " LREM",
187
+ " LRETURN",
188
+ " LSHL",
189
+ " LSHR",
190
+ " LSTORE",
191
+ " LSTORE_0",
192
+ " LSTORE_1",
193
+ " LSTORE_2",
194
+ " LSTORE_3",
195
+ " LSUB",
196
+ " LUSHR",
197
+ " LXOR",
198
+ " MONITORENTER",
199
+ " MONITOREXIT",
200
+ " MULTIANEWARRAY",
201
+ " NEW",
202
+ " NEWARRAY",
203
+ " NOP",
204
+ " POP",
205
+ " POP2",
206
+ " PUTFIELD",
207
+ " PUTSTATIC",
208
+ " RET",
209
+ " RETURN",
210
+ " SALOAD",
211
+ " SASTORE",
212
+ " SIPUSH",
213
+ " SWAP",
214
+ " TABLESWITCH",
215
+ " WIDE",
216
+ "<extra_id_0>",
217
+ "<extra_id_10>",
218
+ "<extra_id_11>",
219
+ "<extra_id_12>",
220
+ "<extra_id_13>",
221
+ "<extra_id_14>",
222
+ "<extra_id_15>",
223
+ "<extra_id_16>",
224
+ "<extra_id_17>",
225
+ "<extra_id_18>",
226
+ "<extra_id_19>",
227
+ "<extra_id_1>",
228
+ "<extra_id_20>",
229
+ "<extra_id_21>",
230
+ "<extra_id_22>",
231
+ "<extra_id_23>",
232
+ "<extra_id_24>",
233
+ "<extra_id_25>",
234
+ "<extra_id_26>",
235
+ "<extra_id_27>",
236
+ "<extra_id_28>",
237
+ "<extra_id_29>",
238
+ "<extra_id_2>",
239
+ "<extra_id_30>",
240
+ "<extra_id_31>",
241
+ "<extra_id_32>",
242
+ "<extra_id_33>",
243
+ "<extra_id_34>",
244
+ "<extra_id_35>",
245
+ "<extra_id_36>",
246
+ "<extra_id_37>",
247
+ "<extra_id_38>",
248
+ "<extra_id_39>",
249
+ "<extra_id_3>",
250
+ "<extra_id_40>",
251
+ "<extra_id_41>",
252
+ "<extra_id_42>",
253
+ "<extra_id_43>",
254
+ "<extra_id_44>",
255
+ "<extra_id_45>",
256
+ "<extra_id_46>",
257
+ "<extra_id_47>",
258
+ "<extra_id_48>",
259
+ "<extra_id_49>",
260
+ "<extra_id_4>",
261
+ "<extra_id_50>",
262
+ "<extra_id_51>",
263
+ "<extra_id_52>",
264
+ "<extra_id_53>",
265
+ "<extra_id_54>",
266
+ "<extra_id_55>",
267
+ "<extra_id_56>",
268
+ "<extra_id_57>",
269
+ "<extra_id_58>",
270
+ "<extra_id_59>",
271
+ "<extra_id_5>",
272
+ "<extra_id_60>",
273
+ "<extra_id_61>",
274
+ "<extra_id_62>",
275
+ "<extra_id_63>",
276
+ "<extra_id_64>",
277
+ "<extra_id_65>",
278
+ "<extra_id_66>",
279
+ "<extra_id_67>",
280
+ "<extra_id_68>",
281
+ "<extra_id_69>",
282
+ "<extra_id_6>",
283
+ "<extra_id_70>",
284
+ "<extra_id_71>",
285
+ "<extra_id_72>",
286
+ "<extra_id_73>",
287
+ "<extra_id_74>",
288
+ "<extra_id_75>",
289
+ "<extra_id_76>",
290
+ "<extra_id_77>",
291
+ "<extra_id_78>",
292
+ "<extra_id_79>",
293
+ "<extra_id_7>",
294
+ "<extra_id_80>",
295
+ "<extra_id_81>",
296
+ "<extra_id_82>",
297
+ "<extra_id_83>",
298
+ "<extra_id_84>",
299
+ "<extra_id_85>",
300
+ "<extra_id_86>",
301
+ "<extra_id_87>",
302
+ "<extra_id_88>",
303
+ "<extra_id_89>",
304
+ "<extra_id_8>",
305
+ "<extra_id_90>",
306
+ "<extra_id_91>",
307
+ "<extra_id_92>",
308
+ "<extra_id_93>",
309
+ "<extra_id_94>",
310
+ "<extra_id_95>",
311
+ "<extra_id_96>",
312
+ "<extra_id_97>",
313
+ "<extra_id_98>",
314
+ "<extra_id_99>",
315
+ "<extra_id_9>"
316
+ ],
317
+ "bos_token": "<s>",
318
+ "cls_token": "<cls>",
319
+ "eos_token": "</s>",
320
+ "mask_token": {
321
+ "content": "<mask>",
322
+ "lstrip": true,
323
+ "normalized": true,
324
+ "rstrip": false,
325
+ "single_word": false
326
+ },
327
+ "pad_token": "<pad>",
328
+ "sep_token": "<sep>",
329
+ "unk_token": "<|UNKNOWN|>"
330
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": {
4
+ "__type": "AddedToken",
5
+ "content": "<s>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false
10
+ },
11
+ "cls_token": {
12
+ "__type": "AddedToken",
13
+ "content": "<s>",
14
+ "lstrip": false,
15
+ "normalized": true,
16
+ "rstrip": false,
17
+ "single_word": false
18
+ },
19
+ "eos_token": {
20
+ "__type": "AddedToken",
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": true,
24
+ "rstrip": false,
25
+ "single_word": false
26
+ },
27
+ "errors": "replace",
28
+ "mask_token": {
29
+ "__type": "AddedToken",
30
+ "content": "<mask>",
31
+ "lstrip": true,
32
+ "normalized": true,
33
+ "rstrip": false,
34
+ "single_word": false
35
+ },
36
+ "model_max_length": 512,
37
+ "name_or_path": "_work/subtokenizer/codet5",
38
+ "pad_token": {
39
+ "__type": "AddedToken",
40
+ "content": "<pad>",
41
+ "lstrip": false,
42
+ "normalized": true,
43
+ "rstrip": false,
44
+ "single_word": false
45
+ },
46
+ "sep_token": {
47
+ "__type": "AddedToken",
48
+ "content": "</s>",
49
+ "lstrip": false,
50
+ "normalized": true,
51
+ "rstrip": false,
52
+ "single_word": false
53
+ },
54
+ "special_tokens_map_file": "/home/pynie/.cache/huggingface/transformers/5941df5e4315c5ab63b7b2ac791fb0bf0f209744a055c06b43b5274849137cdd.b9905d0575bde443a20834122b6e2d48e853b2e36444ce98ddeb43c38097eb3f",
55
+ "tokenizer_class": "RobertaTokenizer",
56
+ "trim_offsets": true,
57
+ "unk_token": {
58
+ "__type": "AddedToken",
59
+ "content": "<unk>",
60
+ "lstrip": false,
61
+ "normalized": true,
62
+ "rstrip": false,
63
+ "single_word": false
64
+ }
65
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff