Upload tokenizer

Browse files

Files changed (6) hide show

added_tokens.json +110 -0
merges.txt +0 -0
special_tokens_map.json +12 -0
tokenizer.json +0 -0
tokenizer_config.json +40 -0
vocab.json +0 -0

added_tokens.json ADDED Viewed

	@@ -0,0 +1,110 @@

+{
+  "<LM>": 50357,
+  "<SC1>": 50358,
+  "<SC2>": 50359,
+  "<SC3>": 50360,
+  "<SC4>": 50361,
+  "<SC5>": 50362,
+  "<SC6>": 50363,
+  "<extra_id_0>": 50257,
+  "<extra_id_10>": 50267,
+  "<extra_id_11>": 50268,
+  "<extra_id_12>": 50269,
+  "<extra_id_13>": 50270,
+  "<extra_id_14>": 50271,
+  "<extra_id_15>": 50272,
+  "<extra_id_16>": 50273,
+  "<extra_id_17>": 50274,
+  "<extra_id_18>": 50275,
+  "<extra_id_19>": 50276,
+  "<extra_id_1>": 50258,
+  "<extra_id_20>": 50277,
+  "<extra_id_21>": 50278,
+  "<extra_id_22>": 50279,
+  "<extra_id_23>": 50280,
+  "<extra_id_24>": 50281,
+  "<extra_id_25>": 50282,
+  "<extra_id_26>": 50283,
+  "<extra_id_27>": 50284,
+  "<extra_id_28>": 50285,
+  "<extra_id_29>": 50286,
+  "<extra_id_2>": 50259,
+  "<extra_id_30>": 50287,
+  "<extra_id_31>": 50288,
+  "<extra_id_32>": 50289,
+  "<extra_id_33>": 50290,
+  "<extra_id_34>": 50291,
+  "<extra_id_35>": 50292,
+  "<extra_id_36>": 50293,
+  "<extra_id_37>": 50294,
+  "<extra_id_38>": 50295,
+  "<extra_id_39>": 50296,
+  "<extra_id_3>": 50260,
+  "<extra_id_40>": 50297,
+  "<extra_id_41>": 50298,
+  "<extra_id_42>": 50299,
+  "<extra_id_43>": 50300,
+  "<extra_id_44>": 50301,
+  "<extra_id_45>": 50302,
+  "<extra_id_46>": 50303,
+  "<extra_id_47>": 50304,
+  "<extra_id_48>": 50305,
+  "<extra_id_49>": 50306,
+  "<extra_id_4>": 50261,
+  "<extra_id_50>": 50307,
+  "<extra_id_51>": 50308,
+  "<extra_id_52>": 50309,
+  "<extra_id_53>": 50310,
+  "<extra_id_54>": 50311,
+  "<extra_id_55>": 50312,
+  "<extra_id_56>": 50313,
+  "<extra_id_57>": 50314,
+  "<extra_id_58>": 50315,
+  "<extra_id_59>": 50316,
+  "<extra_id_5>": 50262,
+  "<extra_id_60>": 50317,
+  "<extra_id_61>": 50318,
+  "<extra_id_62>": 50319,
+  "<extra_id_63>": 50320,
+  "<extra_id_64>": 50321,
+  "<extra_id_65>": 50322,
+  "<extra_id_66>": 50323,
+  "<extra_id_67>": 50324,
+  "<extra_id_68>": 50325,
+  "<extra_id_69>": 50326,
+  "<extra_id_6>": 50263,
+  "<extra_id_70>": 50327,
+  "<extra_id_71>": 50328,
+  "<extra_id_72>": 50329,
+  "<extra_id_73>": 50330,
+  "<extra_id_74>": 50331,
+  "<extra_id_75>": 50332,
+  "<extra_id_76>": 50333,
+  "<extra_id_77>": 50334,
+  "<extra_id_78>": 50335,
+  "<extra_id_79>": 50336,
+  "<extra_id_7>": 50264,
+  "<extra_id_80>": 50337,
+  "<extra_id_81>": 50338,
+  "<extra_id_82>": 50339,
+  "<extra_id_83>": 50340,
+  "<extra_id_84>": 50341,
+  "<extra_id_85>": 50342,
+  "<extra_id_86>": 50343,
+  "<extra_id_87>": 50344,
+  "<extra_id_88>": 50345,
+  "<extra_id_89>": 50346,
+  "<extra_id_8>": 50265,
+  "<extra_id_90>": 50347,
+  "<extra_id_91>": 50348,
+  "<extra_id_92>": 50349,
+  "<extra_id_93>": 50350,
+  "<extra_id_94>": 50351,
+  "<extra_id_95>": 50352,
+  "<extra_id_96>": 50353,
+  "<extra_id_97>": 50354,
+  "<extra_id_98>": 50355,
+  "<extra_id_99>": 50356,
+  "<extra_id_9>": 50266,
+  "<|endoftext|>": 50364
+}

merges.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,12 @@

+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,40 @@

+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff