Upload tokenizer

special_tokens_map.json

@@ -1,5 +1,6 @@
 {
   "additional_special_tokens": [
+    "<s>",
     "<extra_id_0>",
     "<extra_id_1>",
     "<extra_id_2>",
@@ -101,6 +102,13 @@
     "<extra_id_98>",
     "<extra_id_99>"
   ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "eos_token": {
     "content": "</s>",
     "lstrip": false,

spiece.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64164c709b102402833d23138a9013273299a8940358030e1623a16c4e0cb121
-size 969536
+oid sha256:829ed13b97a7ef734aaf095db01da9557aa389fffa99c9a85d483ada98a4bbe6
+size 413481

tokenizer_config.json

@@ -9,6 +9,14 @@
       "single_word": false,
       "special": true
     },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
     "2": {
       "content": "</s>",
       "lstrip": false,
@@ -25,7 +33,7 @@
       "single_word": false,
       "special": true
     },
-    "32000": {
+    "8000": {
       "content": "<extra_id_99>",
       "lstrip": false,
       "normalized": false,
@@ -33,7 +41,7 @@
       "single_word": false,
       "special": true
     },
-    "32001": {
+    "8001": {
       "content": "<extra_id_98>",
       "lstrip": false,
       "normalized": false,
@@ -41,7 +49,7 @@
       "single_word": false,
       "special": true
     },
-    "32002": {
+    "8002": {
       "content": "<extra_id_97>",
       "lstrip": false,
       "normalized": false,
@@ -49,7 +57,7 @@
       "single_word": false,
       "special": true
     },
-    "32003": {
+    "8003": {
       "content": "<extra_id_96>",
       "lstrip": false,
       "normalized": false,
@@ -57,7 +65,7 @@
       "single_word": false,
       "special": true
     },
-    "32004": {
+    "8004": {
       "content": "<extra_id_95>",
       "lstrip": false,
       "normalized": false,
@@ -65,7 +73,7 @@
       "single_word": false,
       "special": true
     },
-    "32005": {
+    "8005": {
       "content": "<extra_id_94>",
       "lstrip": false,
       "normalized": false,
@@ -73,7 +81,7 @@
       "single_word": false,
       "special": true
     },
-    "32006": {
+    "8006": {
       "content": "<extra_id_93>",
       "lstrip": false,
       "normalized": false,
@@ -81,7 +89,7 @@
       "single_word": false,
       "special": true
     },
-    "32007": {
+    "8007": {
       "content": "<extra_id_92>",
       "lstrip": false,
       "normalized": false,
@@ -89,7 +97,7 @@
       "single_word": false,
       "special": true
     },
-    "32008": {
+    "8008": {
       "content": "<extra_id_91>",
       "lstrip": false,
       "normalized": false,
@@ -97,7 +105,7 @@
       "single_word": false,
       "special": true
     },
-    "32009": {
+    "8009": {
       "content": "<extra_id_90>",
       "lstrip": false,
       "normalized": false,
@@ -105,7 +113,7 @@
       "single_word": false,
       "special": true
     },
-    "32010": {
+    "8010": {
       "content": "<extra_id_89>",
       "lstrip": false,
       "normalized": false,
@@ -113,7 +121,7 @@
       "single_word": false,
       "special": true
     },
-    "32011": {
+    "8011": {
       "content": "<extra_id_88>",
       "lstrip": false,
       "normalized": false,
@@ -121,7 +129,7 @@
       "single_word": false,
       "special": true
     },
-    "32012": {
+    "8012": {
       "content": "<extra_id_87>",
       "lstrip": false,
       "normalized": false,
@@ -129,7 +137,7 @@
       "single_word": false,
       "special": true
     },
-    "32013": {
+    "8013": {
       "content": "<extra_id_86>",
       "lstrip": false,
       "normalized": false,
@@ -137,7 +145,7 @@
       "single_word": false,
       "special": true
     },
-    "32014": {
+    "8014": {
       "content": "<extra_id_85>",
       "lstrip": false,
       "normalized": false,
@@ -145,7 +153,7 @@
       "single_word": false,
       "special": true
     },
-    "32015": {
+    "8015": {
       "content": "<extra_id_84>",
       "lstrip": false,
       "normalized": false,
@@ -153,7 +161,7 @@
       "single_word": false,
       "special": true
     },
-    "32016": {
+    "8016": {
       "content": "<extra_id_83>",
       "lstrip": false,
       "normalized": false,
@@ -161,7 +169,7 @@
       "single_word": false,
       "special": true
     },
-    "32017": {
+    "8017": {
       "content": "<extra_id_82>",
       "lstrip": false,
       "normalized": false,
@@ -169,7 +177,7 @@
       "single_word": false,
       "special": true
     },
-    "32018": {
+    "8018": {
       "content": "<extra_id_81>",
       "lstrip": false,
       "normalized": false,
@@ -177,7 +185,7 @@
       "single_word": false,
       "special": true
     },
-    "32019": {
+    "8019": {
       "content": "<extra_id_80>",
       "lstrip": false,
       "normalized": false,
@@ -185,7 +193,7 @@
       "single_word": false,
       "special": true
     },
-    "32020": {
+    "8020": {
       "content": "<extra_id_79>",
       "lstrip": false,
       "normalized": false,
@@ -193,7 +201,7 @@
       "single_word": false,
       "special": true
     },
-    "32021": {
+    "8021": {
       "content": "<extra_id_78>",
       "lstrip": false,
       "normalized": false,
@@ -201,7 +209,7 @@
       "single_word": false,
       "special": true
     },
-    "32022": {
+    "8022": {
       "content": "<extra_id_77>",
       "lstrip": false,
       "normalized": false,
@@ -209,7 +217,7 @@
       "single_word": false,
       "special": true
     },
-    "32023": {
+    "8023": {
       "content": "<extra_id_76>",
       "lstrip": false,
       "normalized": false,
@@ -217,7 +225,7 @@
       "single_word": false,
       "special": true
     },
-    "32024": {
+    "8024": {
       "content": "<extra_id_75>",
       "lstrip": false,
       "normalized": false,
@@ -225,7 +233,7 @@
       "single_word": false,
       "special": true
     },
-    "32025": {
+    "8025": {
       "content": "<extra_id_74>",
       "lstrip": false,
       "normalized": false,
@@ -233,7 +241,7 @@
       "single_word": false,
       "special": true
     },
-    "32026": {
+    "8026": {
       "content": "<extra_id_73>",
       "lstrip": false,
       "normalized": false,
@@ -241,7 +249,7 @@
       "single_word": false,
       "special": true
     },
-    "32027": {
+    "8027": {
       "content": "<extra_id_72>",
       "lstrip": false,
       "normalized": false,
@@ -249,7 +257,7 @@
       "single_word": false,
       "special": true
     },
-    "32028": {
+    "8028": {
       "content": "<extra_id_71>",
       "lstrip": false,
       "normalized": false,
@@ -257,7 +265,7 @@
       "single_word": false,
       "special": true
     },
-    "32029": {
+    "8029": {
       "content": "<extra_id_70>",
       "lstrip": false,
       "normalized": false,
@@ -265,7 +273,7 @@
       "single_word": false,
       "special": true
     },
-    "32030": {
+    "8030": {
       "content": "<extra_id_69>",
       "lstrip": false,
       "normalized": false,
@@ -273,7 +281,7 @@
       "single_word": false,
       "special": true
     },
-    "32031": {
+    "8031": {
       "content": "<extra_id_68>",
       "lstrip": false,
       "normalized": false,
@@ -281,7 +289,7 @@
       "single_word": false,
       "special": true
     },
-    "32032": {
+    "8032": {
       "content": "<extra_id_67>",
       "lstrip": false,
       "normalized": false,
@@ -289,7 +297,7 @@
       "single_word": false,
       "special": true
     },
-    "32033": {
+    "8033": {
       "content": "<extra_id_66>",
       "lstrip": false,
       "normalized": false,
@@ -297,7 +305,7 @@
       "single_word": false,
       "special": true
     },
-    "32034": {
+    "8034": {
       "content": "<extra_id_65>",
       "lstrip": false,
       "normalized": false,
@@ -305,7 +313,7 @@
       "single_word": false,
       "special": true
     },
-    "32035": {
+    "8035": {
       "content": "<extra_id_64>",
       "lstrip": false,
       "normalized": false,
@@ -313,7 +321,7 @@
       "single_word": false,
       "special": true
     },
-    "32036": {
+    "8036": {
       "content": "<extra_id_63>",
       "lstrip": false,
       "normalized": false,
@@ -321,7 +329,7 @@
       "single_word": false,
       "special": true
     },
-    "32037": {
+    "8037": {
       "content": "<extra_id_62>",
       "lstrip": false,
       "normalized": false,
@@ -329,7 +337,7 @@
       "single_word": false,
       "special": true
     },
-    "32038": {
+    "8038": {
       "content": "<extra_id_61>",
       "lstrip": false,
       "normalized": false,
@@ -337,7 +345,7 @@
       "single_word": false,
       "special": true
     },
-    "32039": {
+    "8039": {
       "content": "<extra_id_60>",
       "lstrip": false,
       "normalized": false,
@@ -345,7 +353,7 @@
       "single_word": false,
       "special": true
     },
-    "32040": {
+    "8040": {
       "content": "<extra_id_59>",
       "lstrip": false,
       "normalized": false,
@@ -353,7 +361,7 @@
       "single_word": false,
       "special": true
     },
-    "32041": {
+    "8041": {
       "content": "<extra_id_58>",
       "lstrip": false,
       "normalized": false,
@@ -361,7 +369,7 @@
       "single_word": false,
       "special": true
     },
-    "32042": {
+    "8042": {
       "content": "<extra_id_57>",
       "lstrip": false,
       "normalized": false,
@@ -369,7 +377,7 @@
       "single_word": false,
       "special": true
     },
-    "32043": {
+    "8043": {
       "content": "<extra_id_56>",
       "lstrip": false,
       "normalized": false,
@@ -377,7 +385,7 @@
       "single_word": false,
       "special": true
     },
-    "32044": {
+    "8044": {
       "content": "<extra_id_55>",
       "lstrip": false,
       "normalized": false,
@@ -385,7 +393,7 @@
       "single_word": false,
       "special": true
     },
-    "32045": {
+    "8045": {
       "content": "<extra_id_54>",
       "lstrip": false,
       "normalized": false,
@@ -393,7 +401,7 @@
       "single_word": false,
       "special": true
     },
-    "32046": {
+    "8046": {
       "content": "<extra_id_53>",
       "lstrip": false,
       "normalized": false,
@@ -401,7 +409,7 @@
       "single_word": false,
       "special": true
     },
-    "32047": {
+    "8047": {
       "content": "<extra_id_52>",
       "lstrip": false,
       "normalized": false,
@@ -409,7 +417,7 @@
       "single_word": false,
       "special": true
     },
-    "32048": {
+    "8048": {
       "content": "<extra_id_51>",
       "lstrip": false,
       "normalized": false,
@@ -417,7 +425,7 @@
       "single_word": false,
       "special": true
     },
-    "32049": {
+    "8049": {
       "content": "<extra_id_50>",
       "lstrip": false,
       "normalized": false,
@@ -425,7 +433,7 @@
       "single_word": false,
       "special": true
     },
-    "32050": {
+    "8050": {
       "content": "<extra_id_49>",
       "lstrip": false,
       "normalized": false,
@@ -433,7 +441,7 @@
       "single_word": false,
       "special": true
     },
-    "32051": {
+    "8051": {
       "content": "<extra_id_48>",
       "lstrip": false,
       "normalized": false,
@@ -441,7 +449,7 @@
       "single_word": false,
       "special": true
     },
-    "32052": {
+    "8052": {
       "content": "<extra_id_47>",
       "lstrip": false,
       "normalized": false,
@@ -449,7 +457,7 @@
       "single_word": false,
       "special": true
     },
-    "32053": {
+    "8053": {
       "content": "<extra_id_46>",
       "lstrip": false,
       "normalized": false,
@@ -457,7 +465,7 @@
       "single_word": false,
       "special": true
     },
-    "32054": {
+    "8054": {
       "content": "<extra_id_45>",
       "lstrip": false,
       "normalized": false,
@@ -465,7 +473,7 @@
       "single_word": false,
       "special": true
     },
-    "32055": {
+    "8055": {
       "content": "<extra_id_44>",
       "lstrip": false,
       "normalized": false,
@@ -473,7 +481,7 @@
       "single_word": false,
       "special": true
     },
-    "32056": {
+    "8056": {
       "content": "<extra_id_43>",
       "lstrip": false,
       "normalized": false,
@@ -481,7 +489,7 @@
       "single_word": false,
       "special": true
     },
-    "32057": {
+    "8057": {
       "content": "<extra_id_42>",
       "lstrip": false,
       "normalized": false,
@@ -489,7 +497,7 @@
       "single_word": false,
       "special": true
     },
-    "32058": {
+    "8058": {
       "content": "<extra_id_41>",
       "lstrip": false,
       "normalized": false,
@@ -497,7 +505,7 @@
       "single_word": false,
       "special": true
     },
-    "32059": {
+    "8059": {
       "content": "<extra_id_40>",
       "lstrip": false,
       "normalized": false,
@@ -505,7 +513,7 @@
       "single_word": false,
       "special": true
     },
-    "32060": {
+    "8060": {
       "content": "<extra_id_39>",
       "lstrip": false,
       "normalized": false,
@@ -513,7 +521,7 @@
       "single_word": false,
       "special": true
     },
-    "32061": {
+    "8061": {
       "content": "<extra_id_38>",
       "lstrip": false,
       "normalized": false,
@@ -521,7 +529,7 @@
       "single_word": false,
       "special": true
     },
-    "32062": {
+    "8062": {
       "content": "<extra_id_37>",
       "lstrip": false,
       "normalized": false,
@@ -529,7 +537,7 @@
       "single_word": false,
       "special": true
     },
-    "32063": {
+    "8063": {
       "content": "<extra_id_36>",
       "lstrip": false,
       "normalized": false,
@@ -537,7 +545,7 @@
       "single_word": false,
       "special": true
     },
-    "32064": {
+    "8064": {
       "content": "<extra_id_35>",
       "lstrip": false,
       "normalized": false,
@@ -545,7 +553,7 @@
       "single_word": false,
       "special": true
     },
-    "32065": {
+    "8065": {
       "content": "<extra_id_34>",
       "lstrip": false,
       "normalized": false,
@@ -553,7 +561,7 @@
       "single_word": false,
       "special": true
     },
-    "32066": {
+    "8066": {
       "content": "<extra_id_33>",
       "lstrip": false,
       "normalized": false,
@@ -561,7 +569,7 @@
       "single_word": false,
       "special": true
     },
-    "32067": {
+    "8067": {
       "content": "<extra_id_32>",
       "lstrip": false,
       "normalized": false,
@@ -569,7 +577,7 @@
       "single_word": false,
       "special": true
     },
-    "32068": {
+    "8068": {
       "content": "<extra_id_31>",
       "lstrip": false,
       "normalized": false,
@@ -577,7 +585,7 @@
       "single_word": false,
       "special": true
     },
-    "32069": {
+    "8069": {
       "content": "<extra_id_30>",
       "lstrip": false,
       "normalized": false,
@@ -585,7 +593,7 @@
       "single_word": false,
       "special": true
     },
-    "32070": {
+    "8070": {
       "content": "<extra_id_29>",
       "lstrip": false,
       "normalized": false,
@@ -593,7 +601,7 @@
       "single_word": false,
       "special": true
     },
-    "32071": {
+    "8071": {
       "content": "<extra_id_28>",
       "lstrip": false,
       "normalized": false,
@@ -601,7 +609,7 @@
       "single_word": false,
       "special": true
     },
-    "32072": {
+    "8072": {
       "content": "<extra_id_27>",
       "lstrip": false,
       "normalized": false,
@@ -609,7 +617,7 @@
       "single_word": false,
       "special": true
     },
-    "32073": {
+    "8073": {
       "content": "<extra_id_26>",
       "lstrip": false,
       "normalized": false,
@@ -617,7 +625,7 @@
       "single_word": false,
       "special": true
     },
-    "32074": {
+    "8074": {
       "content": "<extra_id_25>",
       "lstrip": false,
       "normalized": false,
@@ -625,7 +633,7 @@
       "single_word": false,
       "special": true
     },
-    "32075": {
+    "8075": {
       "content": "<extra_id_24>",
       "lstrip": false,
       "normalized": false,
@@ -633,7 +641,7 @@
       "single_word": false,
       "special": true
     },
-    "32076": {
+    "8076": {
       "content": "<extra_id_23>",
       "lstrip": false,
       "normalized": false,
@@ -641,7 +649,7 @@
       "single_word": false,
       "special": true
     },
-    "32077": {
+    "8077": {
       "content": "<extra_id_22>",
       "lstrip": false,
       "normalized": false,
@@ -649,7 +657,7 @@
       "single_word": false,
       "special": true
     },
-    "32078": {
+    "8078": {
       "content": "<extra_id_21>",
       "lstrip": false,
       "normalized": false,
@@ -657,7 +665,7 @@
       "single_word": false,
       "special": true
     },
-    "32079": {
+    "8079": {
       "content": "<extra_id_20>",
       "lstrip": false,
       "normalized": false,
@@ -665,7 +673,7 @@
       "single_word": false,
       "special": true
     },
-    "32080": {
+    "8080": {
       "content": "<extra_id_19>",
       "lstrip": false,
       "normalized": false,
@@ -673,7 +681,7 @@
       "single_word": false,
       "special": true
     },
-    "32081": {
+    "8081": {
       "content": "<extra_id_18>",
       "lstrip": false,
       "normalized": false,
@@ -681,7 +689,7 @@
       "single_word": false,
       "special": true
     },
-    "32082": {
+    "8082": {
       "content": "<extra_id_17>",
       "lstrip": false,
       "normalized": false,
@@ -689,7 +697,7 @@
       "single_word": false,
       "special": true
     },
-    "32083": {
+    "8083": {
       "content": "<extra_id_16>",
       "lstrip": false,
       "normalized": false,
@@ -697,7 +705,7 @@
       "single_word": false,
       "special": true
     },
-    "32084": {
+    "8084": {
       "content": "<extra_id_15>",
       "lstrip": false,
       "normalized": false,
@@ -705,7 +713,7 @@
       "single_word": false,
       "special": true
     },
-    "32085": {
+    "8085": {
       "content": "<extra_id_14>",
       "lstrip": false,
       "normalized": false,
@@ -713,7 +721,7 @@
       "single_word": false,
       "special": true
     },
-    "32086": {
+    "8086": {
       "content": "<extra_id_13>",
       "lstrip": false,
       "normalized": false,
@@ -721,7 +729,7 @@
       "single_word": false,
       "special": true
     },
-    "32087": {
+    "8087": {
       "content": "<extra_id_12>",
       "lstrip": false,
       "normalized": false,
@@ -729,7 +737,7 @@
       "single_word": false,
       "special": true
     },
-    "32088": {
+    "8088": {
       "content": "<extra_id_11>",
       "lstrip": false,
       "normalized": false,
@@ -737,7 +745,7 @@
       "single_word": false,
       "special": true
     },
-    "32089": {
+    "8089": {
       "content": "<extra_id_10>",
       "lstrip": false,
       "normalized": false,
@@ -745,7 +753,7 @@
       "single_word": false,
       "special": true
     },
-    "32090": {
+    "8090": {
       "content": "<extra_id_9>",
       "lstrip": false,
       "normalized": false,
@@ -753,7 +761,7 @@
       "single_word": false,
       "special": true
     },
-    "32091": {
+    "8091": {
       "content": "<extra_id_8>",
       "lstrip": false,
       "normalized": false,
@@ -761,7 +769,7 @@
       "single_word": false,
       "special": true
     },
-    "32092": {
+    "8092": {
       "content": "<extra_id_7>",
       "lstrip": false,
       "normalized": false,
@@ -769,7 +777,7 @@
       "single_word": false,
       "special": true
     },
-    "32093": {
+    "8093": {
       "content": "<extra_id_6>",
       "lstrip": false,
       "normalized": false,
@@ -777,7 +785,7 @@
       "single_word": false,
       "special": true
     },
-    "32094": {
+    "8094": {
       "content": "<extra_id_5>",
       "lstrip": false,
       "normalized": false,
@@ -785,7 +793,7 @@
       "single_word": false,
       "special": true
     },
-    "32095": {
+    "8095": {
       "content": "<extra_id_4>",
       "lstrip": false,
       "normalized": false,
@@ -793,7 +801,7 @@
       "single_word": false,
       "special": true
     },
-    "32096": {
+    "8096": {
       "content": "<extra_id_3>",
       "lstrip": false,
       "normalized": false,
@@ -801,7 +809,7 @@
       "single_word": false,
       "special": true
     },
-    "32097": {
+    "8097": {
       "content": "<extra_id_2>",
       "lstrip": false,
       "normalized": false,
@@ -809,7 +817,7 @@
       "single_word": false,
       "special": true
     },
-    "32098": {
+    "8098": {
       "content": "<extra_id_1>",
       "lstrip": false,
       "normalized": false,
@@ -817,7 +825,7 @@
       "single_word": false,
       "special": true
     },
-    "32099": {
+    "8099": {
       "content": "<extra_id_0>",
       "lstrip": false,
       "normalized": false,
@@ -827,6 +835,7 @@
     }
   },
   "additional_special_tokens": [
+    "<s>",
     "<extra_id_0>",
     "<extra_id_1>",
     "<extra_id_2>",
@@ -928,6 +937,7 @@
     "<extra_id_98>",
     "<extra_id_99>"
   ],
+  "bos_token": "<s>",
   "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",
   "extra_ids": 100,