Upload tokenizer_config.json with huggingface_hub

tokenizer_config.json

@@ -264,30 +264,6 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "100289": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100290": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100291": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
   "additional_special_tokens": [
@@ -325,12 +301,16 @@
     "<|reg7|>",
     "<|extra0|>"
   ],
-  "bos_token": "<s>",
+  "bos_token": "<|endoftext|>",
   "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": true,
-  "eos_token": "</s>",
+  "eos_token": "<|endoftext|>",
+  "max_length": 2048,
   "model_max_length": 2048,
   "pad_token": "<|endoftext|>",
+  "stride": 0,
   "tokenizer_class": "GPT2Tokenizer",
-  "unk_token": "<unk>"
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<|endoftext|>"
 }