Upload tokenizer

spiece.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:829ed13b97a7ef734aaf095db01da9557aa389fffa99c9a85d483ada98a4bbe6
-size 413481
+oid sha256:8674db3129dcc5bae3065822cdbbde6b79939ace7a837cc91ded51c6bc51bba2
+size 321146

tokenizer_config.json

@@ -33,7 +33,7 @@
       "single_word": false,
       "special": true
     },
-    "8000": {
+    "4000": {
       "content": "<extra_id_99>",
       "lstrip": false,
       "normalized": false,
@@ -41,7 +41,7 @@
       "single_word": false,
       "special": true
     },
-    "8001": {
+    "4001": {
       "content": "<extra_id_98>",
       "lstrip": false,
       "normalized": false,
@@ -49,7 +49,7 @@
       "single_word": false,
       "special": true
     },
-    "8002": {
+    "4002": {
       "content": "<extra_id_97>",
       "lstrip": false,
       "normalized": false,
@@ -57,7 +57,7 @@
       "single_word": false,
       "special": true
     },
-    "8003": {
+    "4003": {
       "content": "<extra_id_96>",
       "lstrip": false,
       "normalized": false,
@@ -65,7 +65,7 @@
       "single_word": false,
       "special": true
     },
-    "8004": {
+    "4004": {
       "content": "<extra_id_95>",
       "lstrip": false,
       "normalized": false,
@@ -73,7 +73,7 @@
       "single_word": false,
       "special": true
     },
-    "8005": {
+    "4005": {
       "content": "<extra_id_94>",
       "lstrip": false,
       "normalized": false,
@@ -81,7 +81,7 @@
       "single_word": false,
       "special": true
     },
-    "8006": {
+    "4006": {
       "content": "<extra_id_93>",
       "lstrip": false,
       "normalized": false,
@@ -89,7 +89,7 @@
       "single_word": false,
       "special": true
     },
-    "8007": {
+    "4007": {
       "content": "<extra_id_92>",
       "lstrip": false,
       "normalized": false,
@@ -97,7 +97,7 @@
       "single_word": false,
       "special": true
     },
-    "8008": {
+    "4008": {
       "content": "<extra_id_91>",
       "lstrip": false,
       "normalized": false,
@@ -105,7 +105,7 @@
       "single_word": false,
       "special": true
     },
-    "8009": {
+    "4009": {
       "content": "<extra_id_90>",
       "lstrip": false,
       "normalized": false,
@@ -113,7 +113,7 @@
       "single_word": false,
       "special": true
     },
-    "8010": {
+    "4010": {
       "content": "<extra_id_89>",
       "lstrip": false,
       "normalized": false,
@@ -121,7 +121,7 @@
       "single_word": false,
       "special": true
     },
-    "8011": {
+    "4011": {
       "content": "<extra_id_88>",
       "lstrip": false,
       "normalized": false,
@@ -129,7 +129,7 @@
       "single_word": false,
       "special": true
     },
-    "8012": {
+    "4012": {
       "content": "<extra_id_87>",
       "lstrip": false,
       "normalized": false,
@@ -137,7 +137,7 @@
       "single_word": false,
       "special": true
     },
-    "8013": {
+    "4013": {
       "content": "<extra_id_86>",
       "lstrip": false,
       "normalized": false,
@@ -145,7 +145,7 @@
       "single_word": false,
       "special": true
     },
-    "8014": {
+    "4014": {
       "content": "<extra_id_85>",
       "lstrip": false,
       "normalized": false,
@@ -153,7 +153,7 @@
       "single_word": false,
       "special": true
     },
-    "8015": {
+    "4015": {
       "content": "<extra_id_84>",
       "lstrip": false,
       "normalized": false,
@@ -161,7 +161,7 @@
       "single_word": false,
       "special": true
     },
-    "8016": {
+    "4016": {
       "content": "<extra_id_83>",
       "lstrip": false,
       "normalized": false,
@@ -169,7 +169,7 @@
       "single_word": false,
       "special": true
     },
-    "8017": {
+    "4017": {
       "content": "<extra_id_82>",
       "lstrip": false,
       "normalized": false,
@@ -177,7 +177,7 @@
       "single_word": false,
       "special": true
     },
-    "8018": {
+    "4018": {
       "content": "<extra_id_81>",
       "lstrip": false,
       "normalized": false,
@@ -185,7 +185,7 @@
       "single_word": false,
       "special": true
     },
-    "8019": {
+    "4019": {
       "content": "<extra_id_80>",
       "lstrip": false,
       "normalized": false,
@@ -193,7 +193,7 @@
       "single_word": false,
       "special": true
     },
-    "8020": {
+    "4020": {
       "content": "<extra_id_79>",
       "lstrip": false,
       "normalized": false,
@@ -201,7 +201,7 @@
       "single_word": false,
       "special": true
     },
-    "8021": {
+    "4021": {
       "content": "<extra_id_78>",
       "lstrip": false,
       "normalized": false,
@@ -209,7 +209,7 @@
       "single_word": false,
       "special": true
     },
-    "8022": {
+    "4022": {
       "content": "<extra_id_77>",
       "lstrip": false,
       "normalized": false,
@@ -217,7 +217,7 @@
       "single_word": false,
       "special": true
     },
-    "8023": {
+    "4023": {
       "content": "<extra_id_76>",
       "lstrip": false,
       "normalized": false,
@@ -225,7 +225,7 @@
       "single_word": false,
       "special": true
     },
-    "8024": {
+    "4024": {
       "content": "<extra_id_75>",
       "lstrip": false,
       "normalized": false,
@@ -233,7 +233,7 @@
       "single_word": false,
       "special": true
     },
-    "8025": {
+    "4025": {
       "content": "<extra_id_74>",
       "lstrip": false,
       "normalized": false,
@@ -241,7 +241,7 @@
       "single_word": false,
       "special": true
     },
-    "8026": {
+    "4026": {
       "content": "<extra_id_73>",
       "lstrip": false,
       "normalized": false,
@@ -249,7 +249,7 @@
       "single_word": false,
       "special": true
     },
-    "8027": {
+    "4027": {
       "content": "<extra_id_72>",
       "lstrip": false,
       "normalized": false,
@@ -257,7 +257,7 @@
       "single_word": false,
       "special": true
     },
-    "8028": {
+    "4028": {
       "content": "<extra_id_71>",
       "lstrip": false,
       "normalized": false,
@@ -265,7 +265,7 @@
       "single_word": false,
       "special": true
     },
-    "8029": {
+    "4029": {
       "content": "<extra_id_70>",
       "lstrip": false,
       "normalized": false,
@@ -273,7 +273,7 @@
       "single_word": false,
       "special": true
     },
-    "8030": {
+    "4030": {
       "content": "<extra_id_69>",
       "lstrip": false,
       "normalized": false,
@@ -281,7 +281,7 @@
       "single_word": false,
       "special": true
     },
-    "8031": {
+    "4031": {
       "content": "<extra_id_68>",
       "lstrip": false,
       "normalized": false,
@@ -289,7 +289,7 @@
       "single_word": false,
       "special": true
     },
-    "8032": {
+    "4032": {
       "content": "<extra_id_67>",
       "lstrip": false,
       "normalized": false,
@@ -297,7 +297,7 @@
       "single_word": false,
       "special": true
     },
-    "8033": {
+    "4033": {
       "content": "<extra_id_66>",
       "lstrip": false,
       "normalized": false,
@@ -305,7 +305,7 @@
       "single_word": false,
       "special": true
     },
-    "8034": {
+    "4034": {
       "content": "<extra_id_65>",
       "lstrip": false,
       "normalized": false,
@@ -313,7 +313,7 @@
       "single_word": false,
       "special": true
     },
-    "8035": {
+    "4035": {
       "content": "<extra_id_64>",
       "lstrip": false,
       "normalized": false,
@@ -321,7 +321,7 @@
       "single_word": false,
       "special": true
     },
-    "8036": {
+    "4036": {
       "content": "<extra_id_63>",
       "lstrip": false,
       "normalized": false,
@@ -329,7 +329,7 @@
       "single_word": false,
       "special": true
     },
-    "8037": {
+    "4037": {
       "content": "<extra_id_62>",
       "lstrip": false,
       "normalized": false,
@@ -337,7 +337,7 @@
       "single_word": false,
       "special": true
     },
-    "8038": {
+    "4038": {
       "content": "<extra_id_61>",
       "lstrip": false,
       "normalized": false,
@@ -345,7 +345,7 @@
       "single_word": false,
       "special": true
     },
-    "8039": {
+    "4039": {
       "content": "<extra_id_60>",
       "lstrip": false,
       "normalized": false,
@@ -353,7 +353,7 @@
       "single_word": false,
       "special": true
     },
-    "8040": {
+    "4040": {
       "content": "<extra_id_59>",
       "lstrip": false,
       "normalized": false,
@@ -361,7 +361,7 @@
       "single_word": false,
       "special": true
     },
-    "8041": {
+    "4041": {
       "content": "<extra_id_58>",
       "lstrip": false,
       "normalized": false,
@@ -369,7 +369,7 @@
       "single_word": false,
       "special": true
     },
-    "8042": {
+    "4042": {
       "content": "<extra_id_57>",
       "lstrip": false,
       "normalized": false,
@@ -377,7 +377,7 @@
       "single_word": false,
       "special": true
     },
-    "8043": {
+    "4043": {
       "content": "<extra_id_56>",
       "lstrip": false,
       "normalized": false,
@@ -385,7 +385,7 @@
       "single_word": false,
       "special": true
     },
-    "8044": {
+    "4044": {
       "content": "<extra_id_55>",
       "lstrip": false,
       "normalized": false,
@@ -393,7 +393,7 @@
       "single_word": false,
       "special": true
     },
-    "8045": {
+    "4045": {
       "content": "<extra_id_54>",
       "lstrip": false,
       "normalized": false,
@@ -401,7 +401,7 @@
       "single_word": false,
       "special": true
     },
-    "8046": {
+    "4046": {
       "content": "<extra_id_53>",
       "lstrip": false,
       "normalized": false,
@@ -409,7 +409,7 @@
       "single_word": false,
       "special": true
     },
-    "8047": {
+    "4047": {
       "content": "<extra_id_52>",
       "lstrip": false,
       "normalized": false,
@@ -417,7 +417,7 @@
       "single_word": false,
       "special": true
     },
-    "8048": {
+    "4048": {
       "content": "<extra_id_51>",
       "lstrip": false,
       "normalized": false,
@@ -425,7 +425,7 @@
       "single_word": false,
       "special": true
     },
-    "8049": {
+    "4049": {
       "content": "<extra_id_50>",
       "lstrip": false,
       "normalized": false,
@@ -433,7 +433,7 @@
       "single_word": false,
       "special": true
     },
-    "8050": {
+    "4050": {
       "content": "<extra_id_49>",
       "lstrip": false,
       "normalized": false,
@@ -441,7 +441,7 @@
       "single_word": false,
       "special": true
     },
-    "8051": {
+    "4051": {
       "content": "<extra_id_48>",
       "lstrip": false,
       "normalized": false,
@@ -449,7 +449,7 @@
       "single_word": false,
       "special": true
     },
-    "8052": {
+    "4052": {
       "content": "<extra_id_47>",
       "lstrip": false,
       "normalized": false,
@@ -457,7 +457,7 @@
       "single_word": false,
       "special": true
     },
-    "8053": {
+    "4053": {
       "content": "<extra_id_46>",
       "lstrip": false,
       "normalized": false,
@@ -465,7 +465,7 @@
       "single_word": false,
       "special": true
     },
-    "8054": {
+    "4054": {
       "content": "<extra_id_45>",
       "lstrip": false,
       "normalized": false,
@@ -473,7 +473,7 @@
       "single_word": false,
       "special": true
     },
-    "8055": {
+    "4055": {
       "content": "<extra_id_44>",
       "lstrip": false,
       "normalized": false,
@@ -481,7 +481,7 @@
       "single_word": false,
       "special": true
     },
-    "8056": {
+    "4056": {
       "content": "<extra_id_43>",
       "lstrip": false,
       "normalized": false,
@@ -489,7 +489,7 @@
       "single_word": false,
       "special": true
     },
-    "8057": {
+    "4057": {
       "content": "<extra_id_42>",
       "lstrip": false,
       "normalized": false,
@@ -497,7 +497,7 @@
       "single_word": false,
       "special": true
     },
-    "8058": {
+    "4058": {
       "content": "<extra_id_41>",
       "lstrip": false,
       "normalized": false,
@@ -505,7 +505,7 @@
       "single_word": false,
       "special": true
     },
-    "8059": {
+    "4059": {
       "content": "<extra_id_40>",
       "lstrip": false,
       "normalized": false,
@@ -513,7 +513,7 @@
       "single_word": false,
       "special": true
     },
-    "8060": {
+    "4060": {
       "content": "<extra_id_39>",
       "lstrip": false,
       "normalized": false,
@@ -521,7 +521,7 @@
       "single_word": false,
       "special": true
     },
-    "8061": {
+    "4061": {
       "content": "<extra_id_38>",
       "lstrip": false,
       "normalized": false,
@@ -529,7 +529,7 @@
       "single_word": false,
       "special": true
     },
-    "8062": {
+    "4062": {
       "content": "<extra_id_37>",
       "lstrip": false,
       "normalized": false,
@@ -537,7 +537,7 @@
       "single_word": false,
       "special": true
     },
-    "8063": {
+    "4063": {
       "content": "<extra_id_36>",
       "lstrip": false,
       "normalized": false,
@@ -545,7 +545,7 @@
       "single_word": false,
       "special": true
     },
-    "8064": {
+    "4064": {
       "content": "<extra_id_35>",
       "lstrip": false,
       "normalized": false,
@@ -553,7 +553,7 @@
       "single_word": false,
       "special": true
     },
-    "8065": {
+    "4065": {
       "content": "<extra_id_34>",
       "lstrip": false,
       "normalized": false,
@@ -561,7 +561,7 @@
       "single_word": false,
       "special": true
     },
-    "8066": {
+    "4066": {
       "content": "<extra_id_33>",
       "lstrip": false,
       "normalized": false,
@@ -569,7 +569,7 @@
       "single_word": false,
       "special": true
     },
-    "8067": {
+    "4067": {
       "content": "<extra_id_32>",
       "lstrip": false,
       "normalized": false,
@@ -577,7 +577,7 @@
       "single_word": false,
       "special": true
     },
-    "8068": {
+    "4068": {
       "content": "<extra_id_31>",
       "lstrip": false,
       "normalized": false,
@@ -585,7 +585,7 @@
       "single_word": false,
       "special": true
     },
-    "8069": {
+    "4069": {
       "content": "<extra_id_30>",
       "lstrip": false,
       "normalized": false,
@@ -593,7 +593,7 @@
       "single_word": false,
       "special": true
     },
-    "8070": {
+    "4070": {
       "content": "<extra_id_29>",
       "lstrip": false,
       "normalized": false,
@@ -601,7 +601,7 @@
       "single_word": false,
       "special": true
     },
-    "8071": {
+    "4071": {
       "content": "<extra_id_28>",
       "lstrip": false,
       "normalized": false,
@@ -609,7 +609,7 @@
       "single_word": false,
       "special": true
     },
-    "8072": {
+    "4072": {
       "content": "<extra_id_27>",
       "lstrip": false,
       "normalized": false,
@@ -617,7 +617,7 @@
       "single_word": false,
       "special": true
     },
-    "8073": {
+    "4073": {
       "content": "<extra_id_26>",
       "lstrip": false,
       "normalized": false,
@@ -625,7 +625,7 @@
       "single_word": false,
       "special": true
     },
-    "8074": {
+    "4074": {
       "content": "<extra_id_25>",
       "lstrip": false,
       "normalized": false,
@@ -633,7 +633,7 @@
       "single_word": false,
       "special": true
     },
-    "8075": {
+    "4075": {
       "content": "<extra_id_24>",
       "lstrip": false,
       "normalized": false,
@@ -641,7 +641,7 @@
       "single_word": false,
       "special": true
     },
-    "8076": {
+    "4076": {
       "content": "<extra_id_23>",
       "lstrip": false,
       "normalized": false,
@@ -649,7 +649,7 @@
       "single_word": false,
       "special": true
     },
-    "8077": {
+    "4077": {
       "content": "<extra_id_22>",
       "lstrip": false,
       "normalized": false,
@@ -657,7 +657,7 @@
       "single_word": false,
       "special": true
     },
-    "8078": {
+    "4078": {
       "content": "<extra_id_21>",
       "lstrip": false,
       "normalized": false,
@@ -665,7 +665,7 @@
       "single_word": false,
       "special": true
     },
-    "8079": {
+    "4079": {
       "content": "<extra_id_20>",
       "lstrip": false,
       "normalized": false,
@@ -673,7 +673,7 @@
       "single_word": false,
       "special": true
     },
-    "8080": {
+    "4080": {
       "content": "<extra_id_19>",
       "lstrip": false,
       "normalized": false,
@@ -681,7 +681,7 @@
       "single_word": false,
       "special": true
     },
-    "8081": {
+    "4081": {
       "content": "<extra_id_18>",
       "lstrip": false,
       "normalized": false,
@@ -689,7 +689,7 @@
       "single_word": false,
       "special": true
     },
-    "8082": {
+    "4082": {
       "content": "<extra_id_17>",
       "lstrip": false,
       "normalized": false,
@@ -697,7 +697,7 @@
       "single_word": false,
       "special": true
     },
-    "8083": {
+    "4083": {
       "content": "<extra_id_16>",
       "lstrip": false,
       "normalized": false,
@@ -705,7 +705,7 @@
       "single_word": false,
       "special": true
     },
-    "8084": {
+    "4084": {
       "content": "<extra_id_15>",
       "lstrip": false,
       "normalized": false,
@@ -713,7 +713,7 @@
       "single_word": false,
       "special": true
     },
-    "8085": {
+    "4085": {
       "content": "<extra_id_14>",
       "lstrip": false,
       "normalized": false,
@@ -721,7 +721,7 @@
       "single_word": false,
       "special": true
     },
-    "8086": {
+    "4086": {
       "content": "<extra_id_13>",
       "lstrip": false,
       "normalized": false,
@@ -729,7 +729,7 @@
       "single_word": false,
       "special": true
     },
-    "8087": {
+    "4087": {
       "content": "<extra_id_12>",
       "lstrip": false,
       "normalized": false,
@@ -737,7 +737,7 @@
       "single_word": false,
       "special": true
     },
-    "8088": {
+    "4088": {
       "content": "<extra_id_11>",
       "lstrip": false,
       "normalized": false,
@@ -745,7 +745,7 @@
       "single_word": false,
       "special": true
     },
-    "8089": {
+    "4089": {
       "content": "<extra_id_10>",
       "lstrip": false,
       "normalized": false,
@@ -753,7 +753,7 @@
       "single_word": false,
       "special": true
     },
-    "8090": {
+    "4090": {
       "content": "<extra_id_9>",
       "lstrip": false,
       "normalized": false,
@@ -761,7 +761,7 @@
       "single_word": false,
       "special": true
     },
-    "8091": {
+    "4091": {
       "content": "<extra_id_8>",
       "lstrip": false,
       "normalized": false,
@@ -769,7 +769,7 @@
       "single_word": false,
       "special": true
     },
-    "8092": {
+    "4092": {
       "content": "<extra_id_7>",
       "lstrip": false,
       "normalized": false,
@@ -777,7 +777,7 @@
       "single_word": false,
       "special": true
     },
-    "8093": {
+    "4093": {
       "content": "<extra_id_6>",
       "lstrip": false,
       "normalized": false,
@@ -785,7 +785,7 @@
       "single_word": false,
       "special": true
     },
-    "8094": {
+    "4094": {
       "content": "<extra_id_5>",
       "lstrip": false,
       "normalized": false,
@@ -793,7 +793,7 @@
       "single_word": false,
       "special": true
     },
-    "8095": {
+    "4095": {
       "content": "<extra_id_4>",
       "lstrip": false,
       "normalized": false,
@@ -801,7 +801,7 @@
       "single_word": false,
       "special": true
     },
-    "8096": {
+    "4096": {
       "content": "<extra_id_3>",
       "lstrip": false,
       "normalized": false,
@@ -809,7 +809,7 @@
       "single_word": false,
       "special": true
     },
-    "8097": {
+    "4097": {
       "content": "<extra_id_2>",
       "lstrip": false,
       "normalized": false,
@@ -817,7 +817,7 @@
       "single_word": false,
       "special": true
     },
-    "8098": {
+    "4098": {
       "content": "<extra_id_1>",
       "lstrip": false,
       "normalized": false,
@@ -825,7 +825,7 @@
       "single_word": false,
       "special": true
     },
-    "8099": {
+    "4099": {
       "content": "<extra_id_0>",
       "lstrip": false,
       "normalized": false,