KhimNguyen
/

text2sql

Text2Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

KhimNguyen commited on Apr 22

Commit

d42b110

•

1 Parent(s): 18c55ef

Upload tokenizer

Files changed (1) hide show

tokenizer.json +4 -4

tokenizer.json CHANGED Viewed

@@ -964,8 +964,8 @@
   "pre_tokenizer": {
     "type": "Metaspace",
     "replacement": "▁",
-    "add_prefix_space": true,
-    "prepend_scheme": "always"
   },
   "post_processor": {
     "type": "TemplateProcessing",
@@ -1024,8 +1024,8 @@
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",
-    "add_prefix_space": true,
-    "prepend_scheme": "always"
   },
   "model": {
     "type": "Unigram",

   "pre_tokenizer": {
     "type": "Metaspace",
     "replacement": "▁",
+    "prepend_scheme": "always",
+    "split": true
   },
   "post_processor": {
     "type": "TemplateProcessing",
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",
+    "prepend_scheme": "always",
+    "split": true
   },
   "model": {
     "type": "Unigram",