Upload processor
Browse files- added_tokens.json +1 -6
- special_tokens_map.json +49 -7
added_tokens.json
CHANGED
@@ -6,12 +6,9 @@
|
|
6 |
"</line>": 64006,
|
7 |
"</object>": 64010,
|
8 |
"</phrase>": 64008,
|
9 |
-
"</s>": 2,
|
10 |
"<grounding>": 64012,
|
11 |
"<image>": 64003,
|
12 |
-
"<mask>": 64001,
|
13 |
"<object>": 64009,
|
14 |
-
"<pad>": 1,
|
15 |
"<patch_index_0000>": 64013,
|
16 |
"<patch_index_0001>": 64014,
|
17 |
"<patch_index_0002>": 64015,
|
@@ -1036,7 +1033,5 @@
|
|
1036 |
"<patch_index_1021>": 65034,
|
1037 |
"<patch_index_1022>": 65035,
|
1038 |
"<patch_index_1023>": 65036,
|
1039 |
-
"<phrase>": 64007
|
1040 |
-
"<s>": 0,
|
1041 |
-
"<unk>": 3
|
1042 |
}
|
|
|
6 |
"</line>": 64006,
|
7 |
"</object>": 64010,
|
8 |
"</phrase>": 64008,
|
|
|
9 |
"<grounding>": 64012,
|
10 |
"<image>": 64003,
|
|
|
11 |
"<object>": 64009,
|
|
|
12 |
"<patch_index_0000>": 64013,
|
13 |
"<patch_index_0001>": 64014,
|
14 |
"<patch_index_0002>": 64015,
|
|
|
1033 |
"<patch_index_1021>": 65034,
|
1034 |
"<patch_index_1022>": 65035,
|
1035 |
"<patch_index_1023>": 65036,
|
1036 |
+
"<phrase>": 64007
|
|
|
|
|
1037 |
}
|
special_tokens_map.json
CHANGED
@@ -6,11 +6,53 @@
|
|
6 |
"<unk>",
|
7 |
"<mask>"
|
8 |
],
|
9 |
-
"bos_token":
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
16 |
}
|
|
|
6 |
"<unk>",
|
7 |
"<mask>"
|
8 |
],
|
9 |
+
"bos_token": {
|
10 |
+
"content": "<s>",
|
11 |
+
"lstrip": false,
|
12 |
+
"normalized": false,
|
13 |
+
"rstrip": false,
|
14 |
+
"single_word": false
|
15 |
+
},
|
16 |
+
"cls_token": {
|
17 |
+
"content": "<s>",
|
18 |
+
"lstrip": false,
|
19 |
+
"normalized": false,
|
20 |
+
"rstrip": false,
|
21 |
+
"single_word": false
|
22 |
+
},
|
23 |
+
"eos_token": {
|
24 |
+
"content": "</s>",
|
25 |
+
"lstrip": false,
|
26 |
+
"normalized": false,
|
27 |
+
"rstrip": false,
|
28 |
+
"single_word": false
|
29 |
+
},
|
30 |
+
"mask_token": {
|
31 |
+
"content": "<mask>",
|
32 |
+
"lstrip": true,
|
33 |
+
"normalized": false,
|
34 |
+
"rstrip": false,
|
35 |
+
"single_word": false
|
36 |
+
},
|
37 |
+
"pad_token": {
|
38 |
+
"content": "<pad>",
|
39 |
+
"lstrip": false,
|
40 |
+
"normalized": false,
|
41 |
+
"rstrip": false,
|
42 |
+
"single_word": false
|
43 |
+
},
|
44 |
+
"sep_token": {
|
45 |
+
"content": "</s>",
|
46 |
+
"lstrip": false,
|
47 |
+
"normalized": false,
|
48 |
+
"rstrip": false,
|
49 |
+
"single_word": false
|
50 |
+
},
|
51 |
+
"unk_token": {
|
52 |
+
"content": "<unk>",
|
53 |
+
"lstrip": false,
|
54 |
+
"normalized": false,
|
55 |
+
"rstrip": false,
|
56 |
+
"single_word": false
|
57 |
+
}
|
58 |
}
|