Upload 6 files

special_tokens_map.json

@@ -1,5 +1,14 @@
 {
   "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|pad|>",
+    "<bos>",
+    "<eos>",
+    "<pad>",
+    "<unk>",
+    "<sep>",
+    "<cls>",
+    "<mask>",
     "<greeting>",
     "<farewell>",
     "<thank>",

tokenizer.json

@@ -57,7 +57,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": true,
+      "normalized": false,
       "special": true
     },
     {

tokenizer_config.json

@@ -36,7 +36,7 @@
     "50256": {
       "content": "<|endoftext|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -235,6 +235,15 @@
     }
   },
   "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|pad|>",
+    "<bos>",
+    "<eos>",
+    "<pad>",
+    "<unk>",
+    "<sep>",
+    "<cls>",
+    "<mask>",
     "<greeting>",
     "<farewell>",
     "<thank>",
@@ -245,9 +254,16 @@
   "cls_token": "<cls>",
   "eos_token": "<eos>",
   "mask_token": "<mask>",
+  "max_length": 1024,
   "model_max_length": 1024,
+  "pad_to_multiple_of": null,
   "pad_token": "<eos>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "sep_token": "<sep>",
+  "stride": 0,
   "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }

vocab.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2052635e337c0e877b3d02bbb8e64e3749499b355218dbdb2b74a7285f54da80
-size 8109044
+oid sha256:3ba3c3109ff33976c4bd966589c11ee14fcaa1f4c9e5e154c2ed7f99d80709e7
+size 798156