Commit
•
81d2928
1
Parent(s):
16be035
tokenizer improved
Browse files- tokenizer.json +4 -2
tokenizer.json
CHANGED
|
@@ -104,7 +104,8 @@
|
|
| 104 |
{
|
| 105 |
"type": "Metaspace",
|
| 106 |
"replacement": "▁",
|
| 107 |
-
"add_prefix_space": true
|
|
|
|
| 108 |
}
|
| 109 |
]
|
| 110 |
},
|
|
@@ -186,7 +187,8 @@
|
|
| 186 |
"decoder": {
|
| 187 |
"type": "Metaspace",
|
| 188 |
"replacement": "▁",
|
| 189 |
-
"add_prefix_space": true
|
|
|
|
| 190 |
},
|
| 191 |
"model": {
|
| 192 |
"type": "Unigram",
|
|
|
|
| 104 |
{
|
| 105 |
"type": "Metaspace",
|
| 106 |
"replacement": "▁",
|
| 107 |
+
"add_prefix_space": true,
|
| 108 |
+
"prepend_scheme": "always"
|
| 109 |
}
|
| 110 |
]
|
| 111 |
},
|
|
|
|
| 187 |
"decoder": {
|
| 188 |
"type": "Metaspace",
|
| 189 |
"replacement": "▁",
|
| 190 |
+
"add_prefix_space": true,
|
| 191 |
+
"prepend_scheme": "always"
|
| 192 |
},
|
| 193 |
"model": {
|
| 194 |
"type": "Unigram",
|