add tokenizer

special_tokens_map.json

@@ -0,0 +1 @@
+{"eos_token": "[end]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "mask_token": "[MASK]"}

tokenizer.json

@@ -0,0 +1,290 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 1,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 234,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 235,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 236,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": null,
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "WordPiece",
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": "##",
+    "max_input_chars_per_word": 100,
+    "vocab": {
+      "": 0,
+      "[UNK]": 1,
+      "[start]": 2,
+      "[end]": 3,
+      "malicious": 4,
+      "benign": 5,
+      "alert": 6,
+      "server_name": 7,
+      "sequence": 8,
+      "TLS": 9,
+      "cipher": 10,
+      "change_cipher_spec": 11,
+      "client_key_length": 12,
+      "c0": 13,
+      "s0": 14,
+      "c1": 15,
+      "s1": 16,
+      "c2": 17,
+      "s2": 18,
+      "c3": 19,
+      "s3": 20,
+      "c4": 21,
+      "s4": 22,
+      "c5": 23,
+      "s5": 24,
+      "c6": 25,
+      "s6": 26,
+      "c7": 27,
+      "s7": 28,
+      "c8": 29,
+      "s8": 30,
+      "c9": 31,
+      "s9": 32,
+      "c10": 33,
+      "s10": 34,
+      "c11": 35,
+      "s11": 36,
+      "c12": 37,
+      "s12": 38,
+      "c13": 39,
+      "s13": 40,
+      "c14": 41,
+      "s14": 42,
+      "c15": 43,
+      "s15": 44,
+      "c16": 45,
+      "s16": 46,
+      "l<1": 47,
+      "l:1": 48,
+      "l:2": 49,
+      "l:3": 50,
+      "l:4": 51,
+      "l:5": 52,
+      "l:6": 53,
+      "l:7": 54,
+      "l:8": 55,
+      "l:9": 56,
+      "l:10": 57,
+      "l:11": 58,
+      "l:12": 59,
+      "l:13": 60,
+      "l:14": 61,
+      "l:15": 62,
+      "l:16": 63,
+      "l:17": 64,
+      "l:18": 65,
+      "l:19": 66,
+      "l:20": 67,
+      "l>20": 68,
+      "l>10": 69,
+      "a": 70,
+      "b": 71,
+      "c": 72,
+      "d": 73,
+      "e": 74,
+      "f": 75,
+      "g": 76,
+      "h": 77,
+      "i": 78,
+      "j": 79,
+      "k": 80,
+      "l": 81,
+      "m": 82,
+      "n": 83,
+      "o": 84,
+      "p": 85,
+      "q": 86,
+      "r": 87,
+      "s": 88,
+      "t": 89,
+      "u": 90,
+      "v": 91,
+      "w": 92,
+      "x": 93,
+      "y": 94,
+      "z": 95,
+      "0": 96,
+      "1": 97,
+      "2": 98,
+      "3": 99,
+      "4": 100,
+      "5": 101,
+      "6": 102,
+      "7": 103,
+      "8": 104,
+      "9": 105,
+      ".": 106,
+      "-": 107,
+      "SSLv2": 108,
+      "SSLv3": 109,
+      "TLS1.0": 110,
+      "TLS1.1": 111,
+      "TLS1.2": 112,
+      "TLS1.3": 113,
+      "TLS1.3-d18": 114,
+      "TLS1.3-d19": 115,
+      "TLS_RSA_WITH_RC4_128_MD5": 116,
+      "TLS_RSA_WITH_RC4_128_SHA": 117,
+      "TLS_RSA_WITH_3DES_EDE_CBC_SHA": 118,
+      "TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA": 119,
+      "TLS_DHE_RSA_WITH_DES_CBC_SHA": 120,
+      "TLS_RSA_WITH_AES_128_CBC_SHA": 121,
+      "TLS_DHE_DSS_WITH_AES_128_CBC_SHA": 122,
+      "TLS_DHE_RSA_WITH_AES_128_CBC_SHA": 123,
+      "TLS_RSA_WITH_AES_256_CBC_SHA": 124,
+      "TLS_DHE_DSS_WITH_AES_256_CBC_SHA": 125,
+      "TLS_DHE_RSA_WITH_AES_256_CBC_SHA": 126,
+      "TLS_RSA_WITH_AES_128_CBC_SHA256": 127,
+      "TLS_RSA_WITH_AES_256_CBC_SHA256": 128,
+      "TLS_DHE_DSS_WITH_AES_128_CBC_SHA256": 129,
+      "TLS_RSA_WITH_CAMELLIA_128_CBC_SHA": 130,
+      "TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA": 131,
+      "TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA": 132,
+      "SSL_RSA_EXPORT1024_WITH_RC4_56_SHA": 133,
+      "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256": 134,
+      "TLS_DHE_DSS_WITH_AES_256_CBC_SHA256": 135,
+      "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256": 136,
+      "TLS_RSA_WITH_CAMELLIA_256_CBC_SHA": 137,
+      "TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA": 138,
+      "TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA": 139,
+      "TLS_RSA_WITH_SEED_CBC_SHA": 140,
+      "TLS_RSA_WITH_AES_128_GCM_SHA256": 141,
+      "TLS_RSA_WITH_AES_256_GCM_SHA384": 142,
+      "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256": 143,
+      "TLS_DHE_RSA_WITH_AES_256_GCM_SHA384": 144,
+      "TLS_DHE_DSS_WITH_AES_128_GCM_SHA256": 145,
+      "TLS_DHE_DSS_WITH_AES_256_GCM_SHA384": 146,
+      "TLS_DHE_PSK_WITH_AES_128_GCM_SHA256": 147,
+      "TLS_DHE_PSK_WITH_AES_256_GCM_SHA384": 148,
+      "TLS_AES_128_GCM_SHA256": 149,
+      "TLS_AES_256_GCM_SHA384": 150,
+      "TLS_CHACHA20_POLY1305_SHA256": 151,
+      "TLS_AES_128_CCM_SHA256": 152,
+      "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA": 153,
+      "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA": 154,
+      "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA": 155,
+      "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA": 156,
+      "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA": 157,
+      "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256": 158,
+      "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384": 159,
+      "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256": 160,
+      "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384": 161,
+      "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": 162,
+      "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": 163,
+      "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256": 164,
+      "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384": 165,
+      "TLS_DHE_RSA_WITH_AES_128_CCM": 166,
+      "TLS_DHE_RSA_WITH_AES_256_CCM": 167,
+      "TLS_DHE_RSA_WITH_AES_128_CCM_8": 168,
+      "TLS_DHE_RSA_WITH_AES_256_CCM_8": 169,
+      "TLS_DHE_PSK_WITH_AES_128_CCM": 170,
+      "TLS_DHE_PSK_WITH_AES_256_CCM": 171,
+      "TLS_ECDHE_ECDSA_WITH_AES_128_CCM": 172,
+      "TLS_ECDHE_ECDSA_WITH_AES_256_CCM": 173,
+      "TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8": 174,
+      "TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8": 175,
+      "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256": 176,
+      "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256": 177,
+      "TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256": 178,
+      "TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256": 179,
+      "TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256": 180,
+      "TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256": 181,
+      "TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384": 182,
+      "TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256": 183,
+      "FIN": 184,
+      "SYN": 185,
+      "RST": 186,
+      "PSH": 187,
+      "ACK": 188,
+      "URG": 189,
+      "ECE": 190,
+      "CWR": 191,
+      "ramnit": 192,
+      "crthrazy": 193,
+      "nymaim": 194,
+      "bunitu": 195,
+      "zeus": 196,
+      "azorult": 197,
+      "parite": 198,
+      "vawtrak": 199,
+      "reposfxg": 200,
+      "zeus-panda": 201,
+      "gandcrab": 202,
+      "bankerx": 203,
+      "gootkit": 204,
+      "dridex": 205,
+      "upatre": 206,
+      "qakbot": 207,
+      "chthonic": 208,
+      "emotet": 209,
+      "troldesh": 210,
+      "kovter": 211,
+      "boleto": 212,
+      "hancitor": 213,
+      "remcos": 214,
+      "trickbot": 215,
+      "rig": 216,
+      "tofsee": 217,
+      "neutrino": 218,
+      "icedid": 219,
+      "dreambot": 220,
+      "miuref": 221,
+      "crypt": 222,
+      "cerber": 223,
+      "unclassified": 224,
+      "sigma": 225,
+      "spora": 226,
+      "locky": 227,
+      "fallout": 228,
+      "banload": 229,
+      "globeimposter": 230,
+      "angler": 231,
+      "ursnif": 232,
+      "?": 233,
+      "[PAD]": 234
+    }
+  }
+}

tokenizer_config.json

@@ -0,0 +1 @@
+{"max_len": 256, "tokenizer_class": "PreTrainedTokenizerFast"}