KhimNguyen
commited on
Commit
•
5a74898
1
Parent(s):
3222008
Upload tokenizer
Browse files- tokenizer.json +4 -4
tokenizer.json
CHANGED
@@ -964,8 +964,8 @@
|
|
964 |
"pre_tokenizer": {
|
965 |
"type": "Metaspace",
|
966 |
"replacement": "▁",
|
967 |
-
"
|
968 |
-
"
|
969 |
},
|
970 |
"post_processor": {
|
971 |
"type": "TemplateProcessing",
|
@@ -1024,8 +1024,8 @@
|
|
1024 |
"decoder": {
|
1025 |
"type": "Metaspace",
|
1026 |
"replacement": "▁",
|
1027 |
-
"
|
1028 |
-
"
|
1029 |
},
|
1030 |
"model": {
|
1031 |
"type": "Unigram",
|
|
|
964 |
"pre_tokenizer": {
|
965 |
"type": "Metaspace",
|
966 |
"replacement": "▁",
|
967 |
+
"prepend_scheme": "always",
|
968 |
+
"split": true
|
969 |
},
|
970 |
"post_processor": {
|
971 |
"type": "TemplateProcessing",
|
|
|
1024 |
"decoder": {
|
1025 |
"type": "Metaspace",
|
1026 |
"replacement": "▁",
|
1027 |
+
"prepend_scheme": "always",
|
1028 |
+
"split": true
|
1029 |
},
|
1030 |
"model": {
|
1031 |
"type": "Unigram",
|