Commit
•
e00e392
1
Parent(s):
1c46b75
tokenizer improved
Browse files- tokenizer.json +3 -1
tokenizer.json
CHANGED
|
@@ -91,6 +91,7 @@
|
|
| 91 |
"pre_tokenizer": {
|
| 92 |
"type": "Metaspace",
|
| 93 |
"replacement": "▁",
|
|
|
|
| 94 |
"prepend_scheme": "never",
|
| 95 |
"split": true
|
| 96 |
},
|
|
@@ -151,6 +152,7 @@
|
|
| 151 |
"decoder": {
|
| 152 |
"type": "Metaspace",
|
| 153 |
"replacement": "▁",
|
|
|
|
| 154 |
"prepend_scheme": "never",
|
| 155 |
"split": true
|
| 156 |
},
|
|
@@ -177365,4 +177367,4 @@
|
|
| 177365 |
],
|
| 177366 |
"byte_fallback": false
|
| 177367 |
}
|
| 177368 |
-
}
|
|
|
|
| 91 |
"pre_tokenizer": {
|
| 92 |
"type": "Metaspace",
|
| 93 |
"replacement": "▁",
|
| 94 |
+
"add_prefix_space": false,
|
| 95 |
"prepend_scheme": "never",
|
| 96 |
"split": true
|
| 97 |
},
|
|
|
|
| 152 |
"decoder": {
|
| 153 |
"type": "Metaspace",
|
| 154 |
"replacement": "▁",
|
| 155 |
+
"add_prefix_space": false,
|
| 156 |
"prepend_scheme": "never",
|
| 157 |
"split": true
|
| 158 |
},
|
|
|
|
| 177367 |
],
|
| 177368 |
"byte_fallback": false
|
| 177369 |
}
|
| 177370 |
+
}
|