Upload tokenizer
Browse files- tokenizer.json +9 -0
- tokenizer_config.json +8 -0
tokenizer.json
CHANGED
|
@@ -16016,6 +16016,15 @@
|
|
| 16016 |
"rstrip": false,
|
| 16017 |
"normalized": true,
|
| 16018 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16019 |
}
|
| 16020 |
],
|
| 16021 |
"normalizer": null,
|
|
|
|
| 16016 |
"rstrip": false,
|
| 16017 |
"normalized": true,
|
| 16018 |
"special": false
|
| 16019 |
+
},
|
| 16020 |
+
{
|
| 16021 |
+
"id": 3490,
|
| 16022 |
+
"content": "sub",
|
| 16023 |
+
"single_word": false,
|
| 16024 |
+
"lstrip": false,
|
| 16025 |
+
"rstrip": false,
|
| 16026 |
+
"normalized": true,
|
| 16027 |
+
"special": false
|
| 16028 |
}
|
| 16029 |
],
|
| 16030 |
"normalizer": null,
|
tokenizer_config.json
CHANGED
|
@@ -14223,6 +14223,14 @@
|
|
| 14223 |
"rstrip": false,
|
| 14224 |
"single_word": false,
|
| 14225 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14226 |
}
|
| 14227 |
},
|
| 14228 |
"bos_token": "</s>",
|
|
|
|
| 14223 |
"rstrip": false,
|
| 14224 |
"single_word": false,
|
| 14225 |
"special": false
|
| 14226 |
+
},
|
| 14227 |
+
"3490": {
|
| 14228 |
+
"content": "sub",
|
| 14229 |
+
"lstrip": false,
|
| 14230 |
+
"normalized": true,
|
| 14231 |
+
"rstrip": false,
|
| 14232 |
+
"single_word": false,
|
| 14233 |
+
"special": false
|
| 14234 |
}
|
| 14235 |
},
|
| 14236 |
"bos_token": "</s>",
|