Upload tokenizer.json with huggingface_hub
Browse files- tokenizer.json +1 -3
tokenizer.json
CHANGED
@@ -991,7 +991,6 @@
|
|
991 |
{
|
992 |
"type": "Metaspace",
|
993 |
"replacement": "▁",
|
994 |
-
"add_prefix_space": true,
|
995 |
"prepend_scheme": "always"
|
996 |
},
|
997 |
{
|
@@ -1105,8 +1104,7 @@
|
|
1105 |
{
|
1106 |
"type": "Metaspace",
|
1107 |
"replacement": "▁",
|
1108 |
-
"
|
1109 |
-
"prepend_scheme": "always"
|
1110 |
},
|
1111 |
{
|
1112 |
"type": "Strip",
|
|
|
991 |
{
|
992 |
"type": "Metaspace",
|
993 |
"replacement": "▁",
|
|
|
994 |
"prepend_scheme": "always"
|
995 |
},
|
996 |
{
|
|
|
1104 |
{
|
1105 |
"type": "Metaspace",
|
1106 |
"replacement": "▁",
|
1107 |
+
"prepend_scheme": "never"
|
|
|
1108 |
},
|
1109 |
{
|
1110 |
"type": "Strip",
|