ydshieh HF staff commited on
Commit
8bc83fb
1 Parent(s): aaa2bff

Upload processor

Browse files
Files changed (2) hide show
  1. added_tokens.json +1 -6
  2. special_tokens_map.json +49 -7
added_tokens.json CHANGED
@@ -6,12 +6,9 @@
6
  "</line>": 64006,
7
  "</object>": 64010,
8
  "</phrase>": 64008,
9
- "</s>": 2,
10
  "<grounding>": 64012,
11
  "<image>": 64003,
12
- "<mask>": 64001,
13
  "<object>": 64009,
14
- "<pad>": 1,
15
  "<patch_index_0000>": 64013,
16
  "<patch_index_0001>": 64014,
17
  "<patch_index_0002>": 64015,
@@ -1036,7 +1033,5 @@
1036
  "<patch_index_1021>": 65034,
1037
  "<patch_index_1022>": 65035,
1038
  "<patch_index_1023>": 65036,
1039
- "<phrase>": 64007,
1040
- "<s>": 0,
1041
- "<unk>": 3
1042
  }
 
6
  "</line>": 64006,
7
  "</object>": 64010,
8
  "</phrase>": 64008,
 
9
  "<grounding>": 64012,
10
  "<image>": 64003,
 
11
  "<object>": 64009,
 
12
  "<patch_index_0000>": 64013,
13
  "<patch_index_0001>": 64014,
14
  "<patch_index_0002>": 64015,
 
1033
  "<patch_index_1021>": 65034,
1034
  "<patch_index_1022>": 65035,
1035
  "<patch_index_1023>": 65036,
1036
+ "<phrase>": 64007
 
 
1037
  }
special_tokens_map.json CHANGED
@@ -6,11 +6,53 @@
6
  "<unk>",
7
  "<mask>"
8
  ],
9
- "bos_token": "<s>",
10
- "cls_token": "<s>",
11
- "eos_token": "</s>",
12
- "mask_token": "<mask>",
13
- "pad_token": "<pad>",
14
- "sep_token": "</s>",
15
- "unk_token": "<unk>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  }
 
6
  "<unk>",
7
  "<mask>"
8
  ],
9
+ "bos_token": {
10
+ "content": "<s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "cls_token": {
17
+ "content": "<s>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "eos_token": {
24
+ "content": "</s>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "mask_token": {
31
+ "content": "<mask>",
32
+ "lstrip": true,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ },
37
+ "pad_token": {
38
+ "content": "<pad>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false
43
+ },
44
+ "sep_token": {
45
+ "content": "</s>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false
50
+ },
51
+ "unk_token": {
52
+ "content": "<unk>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false
57
+ }
58
  }