Hugging Face's logo

JuIm
/
Protein-Sequence-Generation-Tokenizer

Transformers
Model card Files Community
JuIm commited on
Commit
ae175d3
·
verified ·
1 Parent(s): 63d0ebc

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -0
  2. tokenizer_config.json +57 -49
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -1,190 +1,198 @@
1
  {
2
  "added_tokens_decoder": {
3
  "0": {
4
- "content": "R",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
8
  "single_word": false,
9
- "special": false
10
  },
11
  "1": {
12
- "content": "H",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
16
  "single_word": false,
17
- "special": false
18
  },
19
  "2": {
20
- "content": "K",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
24
  "single_word": false,
25
- "special": false
26
  },
27
  "3": {
28
- "content": "D",
 
 
 
 
 
 
 
 
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
32
  "single_word": false,
33
  "special": false
34
  },
35
- "4": {
36
- "content": "E",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
40
  "single_word": false,
41
  "special": false
42
  },
43
- "5": {
44
- "content": "S",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
48
  "single_word": false,
49
  "special": false
50
  },
51
- "6": {
52
- "content": "T",
53
  "lstrip": false,
54
  "normalized": false,
55
  "rstrip": false,
56
  "single_word": false,
57
  "special": false
58
  },
59
- "7": {
60
- "content": "N",
61
  "lstrip": false,
62
  "normalized": false,
63
  "rstrip": false,
64
  "single_word": false,
65
  "special": false
66
  },
67
- "8": {
68
- "content": "Q",
69
  "lstrip": false,
70
  "normalized": false,
71
  "rstrip": false,
72
  "single_word": false,
73
  "special": false
74
  },
75
- "9": {
76
- "content": "C",
77
  "lstrip": false,
78
  "normalized": false,
79
  "rstrip": false,
80
  "single_word": false,
81
  "special": false
82
  },
83
- "10": {
84
- "content": "G",
85
  "lstrip": false,
86
  "normalized": false,
87
  "rstrip": false,
88
  "single_word": false,
89
  "special": false
90
  },
91
- "11": {
92
- "content": "P",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,
96
  "single_word": false,
97
  "special": false
98
  },
99
- "12": {
100
- "content": "A",
101
  "lstrip": false,
102
  "normalized": false,
103
  "rstrip": false,
104
  "single_word": false,
105
  "special": false
106
  },
107
- "13": {
108
- "content": "V",
109
  "lstrip": false,
110
  "normalized": false,
111
  "rstrip": false,
112
  "single_word": false,
113
  "special": false
114
  },
115
- "14": {
116
- "content": "I",
117
  "lstrip": false,
118
  "normalized": false,
119
  "rstrip": false,
120
  "single_word": false,
121
  "special": false
122
  },
123
- "15": {
124
- "content": "L",
125
  "lstrip": false,
126
  "normalized": false,
127
  "rstrip": false,
128
  "single_word": false,
129
  "special": false
130
  },
131
- "16": {
132
- "content": "M",
133
  "lstrip": false,
134
  "normalized": false,
135
  "rstrip": false,
136
  "single_word": false,
137
  "special": false
138
  },
139
- "17": {
140
- "content": "F",
141
  "lstrip": false,
142
  "normalized": false,
143
  "rstrip": false,
144
  "single_word": false,
145
  "special": false
146
  },
147
- "18": {
148
- "content": "Y",
149
  "lstrip": false,
150
  "normalized": false,
151
  "rstrip": false,
152
  "single_word": false,
153
  "special": false
154
  },
155
- "19": {
156
- "content": "W",
157
  "lstrip": false,
158
  "normalized": false,
159
  "rstrip": false,
160
  "single_word": false,
161
  "special": false
162
  },
163
- "20": {
164
- "content": "<bos>",
165
  "lstrip": false,
166
  "normalized": false,
167
  "rstrip": false,
168
  "single_word": false,
169
- "special": true
170
  },
171
- "21": {
172
- "content": "<eos>",
173
  "lstrip": false,
174
  "normalized": false,
175
  "rstrip": false,
176
  "single_word": false,
177
- "special": true
178
  },
179
- "22": {
180
- "content": "<pad>",
181
  "lstrip": false,
182
  "normalized": false,
183
  "rstrip": false,
184
  "single_word": false,
185
- "special": true
186
  },
187
- "23": {
188
  "content": "<mask>",
189
  "lstrip": false,
190
  "normalized": false,
 
1
  {
2
  "added_tokens_decoder": {
3
  "0": {
4
+ "content": "<unk>",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
8
  "single_word": false,
9
+ "special": true
10
  },
11
  "1": {
12
+ "content": "<bos>",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
16
  "single_word": false,
17
+ "special": true
18
  },
19
  "2": {
20
+ "content": "<eos>",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
24
  "single_word": false,
25
+ "special": true
26
  },
27
  "3": {
28
+ "content": "<pad>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "26": {
36
+ "content": "A",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
40
  "single_word": false,
41
  "special": false
42
  },
43
+ "28": {
44
+ "content": "C",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
48
  "single_word": false,
49
  "special": false
50
  },
51
+ "29": {
52
+ "content": "D",
53
  "lstrip": false,
54
  "normalized": false,
55
  "rstrip": false,
56
  "single_word": false,
57
  "special": false
58
  },
59
+ "30": {
60
+ "content": "E",
61
  "lstrip": false,
62
  "normalized": false,
63
  "rstrip": false,
64
  "single_word": false,
65
  "special": false
66
  },
67
+ "31": {
68
+ "content": "F",
69
  "lstrip": false,
70
  "normalized": false,
71
  "rstrip": false,
72
  "single_word": false,
73
  "special": false
74
  },
75
+ "32": {
76
+ "content": "G",
77
  "lstrip": false,
78
  "normalized": false,
79
  "rstrip": false,
80
  "single_word": false,
81
  "special": false
82
  },
83
+ "33": {
84
+ "content": "H",
85
  "lstrip": false,
86
  "normalized": false,
87
  "rstrip": false,
88
  "single_word": false,
89
  "special": false
90
  },
91
+ "34": {
92
+ "content": "I",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,
96
  "single_word": false,
97
  "special": false
98
  },
99
+ "36": {
100
+ "content": "K",
101
  "lstrip": false,
102
  "normalized": false,
103
  "rstrip": false,
104
  "single_word": false,
105
  "special": false
106
  },
107
+ "37": {
108
+ "content": "L",
109
  "lstrip": false,
110
  "normalized": false,
111
  "rstrip": false,
112
  "single_word": false,
113
  "special": false
114
  },
115
+ "38": {
116
+ "content": "M",
117
  "lstrip": false,
118
  "normalized": false,
119
  "rstrip": false,
120
  "single_word": false,
121
  "special": false
122
  },
123
+ "39": {
124
+ "content": "N",
125
  "lstrip": false,
126
  "normalized": false,
127
  "rstrip": false,
128
  "single_word": false,
129
  "special": false
130
  },
131
+ "41": {
132
+ "content": "P",
133
  "lstrip": false,
134
  "normalized": false,
135
  "rstrip": false,
136
  "single_word": false,
137
  "special": false
138
  },
139
+ "42": {
140
+ "content": "Q",
141
  "lstrip": false,
142
  "normalized": false,
143
  "rstrip": false,
144
  "single_word": false,
145
  "special": false
146
  },
147
+ "43": {
148
+ "content": "R",
149
  "lstrip": false,
150
  "normalized": false,
151
  "rstrip": false,
152
  "single_word": false,
153
  "special": false
154
  },
155
+ "44": {
156
+ "content": "S",
157
  "lstrip": false,
158
  "normalized": false,
159
  "rstrip": false,
160
  "single_word": false,
161
  "special": false
162
  },
163
+ "45": {
164
+ "content": "T",
165
  "lstrip": false,
166
  "normalized": false,
167
  "rstrip": false,
168
  "single_word": false,
169
  "special": false
170
  },
171
+ "47": {
172
+ "content": "V",
173
  "lstrip": false,
174
  "normalized": false,
175
  "rstrip": false,
176
  "single_word": false,
177
+ "special": false
178
  },
179
+ "48": {
180
+ "content": "W",
181
  "lstrip": false,
182
  "normalized": false,
183
  "rstrip": false,
184
  "single_word": false,
185
+ "special": false
186
  },
187
+ "50": {
188
+ "content": "Y",
189
  "lstrip": false,
190
  "normalized": false,
191
  "rstrip": false,
192
  "single_word": false,
193
+ "special": false
194
  },
195
+ "30000": {
196
  "content": "<mask>",
197
  "lstrip": false,
198
  "normalized": false,