add tokenizer

tokenizer.json

@@ -1,290 +1 @@
-{
-  "version": "1.0",
-  "truncation": null,
-  "padding": null,
-  "added_tokens": [
-    {
-      "id": 1,
-      "content": "[UNK]",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 234,
-      "content": "[PAD]",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 235,
-      "content": "[MASK]",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 236,
-      "content": "[SEP]",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    }
-  ],
-  "normalizer": null,
-  "pre_tokenizer": null,
-  "post_processor": null,
-  "decoder": null,
-  "model": {
-    "type": "WordPiece",
-    "unk_token": "[UNK]",
-    "continuing_subword_prefix": "##",
-    "max_input_chars_per_word": 100,
-    "vocab": {
-      "": 0,
-      "[UNK]": 1,
-      "[start]": 2,
-      "[end]": 3,
-      "malicious": 4,
-      "benign": 5,
-      "alert": 6,
-      "server_name": 7,
-      "sequence": 8,
-      "TLS": 9,
-      "cipher": 10,
-      "change_cipher_spec": 11,
-      "client_key_length": 12,
-      "c0": 13,
-      "s0": 14,
-      "c1": 15,
-      "s1": 16,
-      "c2": 17,
-      "s2": 18,
-      "c3": 19,
-      "s3": 20,
-      "c4": 21,
-      "s4": 22,
-      "c5": 23,
-      "s5": 24,
-      "c6": 25,
-      "s6": 26,
-      "c7": 27,
-      "s7": 28,
-      "c8": 29,
-      "s8": 30,
-      "c9": 31,
-      "s9": 32,
-      "c10": 33,
-      "s10": 34,
-      "c11": 35,
-      "s11": 36,
-      "c12": 37,
-      "s12": 38,
-      "c13": 39,
-      "s13": 40,
-      "c14": 41,
-      "s14": 42,
-      "c15": 43,
-      "s15": 44,
-      "c16": 45,
-      "s16": 46,
-      "l<1": 47,
-      "l:1": 48,
-      "l:2": 49,
-      "l:3": 50,
-      "l:4": 51,
-      "l:5": 52,
-      "l:6": 53,
-      "l:7": 54,
-      "l:8": 55,
-      "l:9": 56,
-      "l:10": 57,
-      "l:11": 58,
-      "l:12": 59,
-      "l:13": 60,
-      "l:14": 61,
-      "l:15": 62,
-      "l:16": 63,
-      "l:17": 64,
-      "l:18": 65,
-      "l:19": 66,
-      "l:20": 67,
-      "l>20": 68,
-      "l>10": 69,
-      "a": 70,
-      "b": 71,
-      "c": 72,
-      "d": 73,
-      "e": 74,
-      "f": 75,
-      "g": 76,
-      "h": 77,
-      "i": 78,
-      "j": 79,
-      "k": 80,
-      "l": 81,
-      "m": 82,
-      "n": 83,
-      "o": 84,
-      "p": 85,
-      "q": 86,
-      "r": 87,
-      "s": 88,
-      "t": 89,
-      "u": 90,
-      "v": 91,
-      "w": 92,
-      "x": 93,
-      "y": 94,
-      "z": 95,
-      "0": 96,
-      "1": 97,
-      "2": 98,
-      "3": 99,
-      "4": 100,
-      "5": 101,
-      "6": 102,
-      "7": 103,
-      "8": 104,
-      "9": 105,
-      ".": 106,
-      "-": 107,
-      "SSLv2": 108,
-      "SSLv3": 109,
-      "TLS1.0": 110,
-      "TLS1.1": 111,
-      "TLS1.2": 112,
-      "TLS1.3": 113,
-      "TLS1.3-d18": 114,
-      "TLS1.3-d19": 115,
-      "TLS_RSA_WITH_RC4_128_MD5": 116,
-      "TLS_RSA_WITH_RC4_128_SHA": 117,
-      "TLS_RSA_WITH_3DES_EDE_CBC_SHA": 118,
-      "TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA": 119,
-      "TLS_DHE_RSA_WITH_DES_CBC_SHA": 120,
-      "TLS_RSA_WITH_AES_128_CBC_SHA": 121,
-      "TLS_DHE_DSS_WITH_AES_128_CBC_SHA": 122,
-      "TLS_DHE_RSA_WITH_AES_128_CBC_SHA": 123,
-      "TLS_RSA_WITH_AES_256_CBC_SHA": 124,
-      "TLS_DHE_DSS_WITH_AES_256_CBC_SHA": 125,
-      "TLS_DHE_RSA_WITH_AES_256_CBC_SHA": 126,
-      "TLS_RSA_WITH_AES_128_CBC_SHA256": 127,
-      "TLS_RSA_WITH_AES_256_CBC_SHA256": 128,
-      "TLS_DHE_DSS_WITH_AES_128_CBC_SHA256": 129,
-      "TLS_RSA_WITH_CAMELLIA_128_CBC_SHA": 130,
-      "TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA": 131,
-      "TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA": 132,
-      "SSL_RSA_EXPORT1024_WITH_RC4_56_SHA": 133,
-      "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256": 134,
-      "TLS_DHE_DSS_WITH_AES_256_CBC_SHA256": 135,
-      "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256": 136,
-      "TLS_RSA_WITH_CAMELLIA_256_CBC_SHA": 137,
-      "TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA": 138,
-      "TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA": 139,
-      "TLS_RSA_WITH_SEED_CBC_SHA": 140,
-      "TLS_RSA_WITH_AES_128_GCM_SHA256": 141,
-      "TLS_RSA_WITH_AES_256_GCM_SHA384": 142,
-      "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256": 143,
-      "TLS_DHE_RSA_WITH_AES_256_GCM_SHA384": 144,
-      "TLS_DHE_DSS_WITH_AES_128_GCM_SHA256": 145,
-      "TLS_DHE_DSS_WITH_AES_256_GCM_SHA384": 146,
-      "TLS_DHE_PSK_WITH_AES_128_GCM_SHA256": 147,
-      "TLS_DHE_PSK_WITH_AES_256_GCM_SHA384": 148,
-      "TLS_AES_128_GCM_SHA256": 149,
-      "TLS_AES_256_GCM_SHA384": 150,
-      "TLS_CHACHA20_POLY1305_SHA256": 151,
-      "TLS_AES_128_CCM_SHA256": 152,
-      "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA": 153,
-      "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA": 154,
-      "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA": 155,
-      "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA": 156,
-      "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA": 157,
-      "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256": 158,
-      "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384": 159,
-      "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256": 160,
-      "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384": 161,
-      "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": 162,
-      "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": 163,
-      "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256": 164,
-      "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384": 165,
-      "TLS_DHE_RSA_WITH_AES_128_CCM": 166,
-      "TLS_DHE_RSA_WITH_AES_256_CCM": 167,
-      "TLS_DHE_RSA_WITH_AES_128_CCM_8": 168,
-      "TLS_DHE_RSA_WITH_AES_256_CCM_8": 169,
-      "TLS_DHE_PSK_WITH_AES_128_CCM": 170,
-      "TLS_DHE_PSK_WITH_AES_256_CCM": 171,
-      "TLS_ECDHE_ECDSA_WITH_AES_128_CCM": 172,
-      "TLS_ECDHE_ECDSA_WITH_AES_256_CCM": 173,
-      "TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8": 174,
-      "TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8": 175,
-      "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256": 176,
-      "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256": 177,
-      "TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256": 178,
-      "TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256": 179,
-      "TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256": 180,
-      "TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256": 181,
-      "TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384": 182,
-      "TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256": 183,
-      "FIN": 184,
-      "SYN": 185,
-      "RST": 186,
-      "PSH": 187,
-      "ACK": 188,
-      "URG": 189,
-      "ECE": 190,
-      "CWR": 191,
-      "ramnit": 192,
-      "crthrazy": 193,
-      "nymaim": 194,
-      "bunitu": 195,
-      "zeus": 196,
-      "azorult": 197,
-      "parite": 198,
-      "vawtrak": 199,
-      "reposfxg": 200,
-      "zeus-panda": 201,
-      "gandcrab": 202,
-      "bankerx": 203,
-      "gootkit": 204,
-      "dridex": 205,
-      "upatre": 206,
-      "qakbot": 207,
-      "chthonic": 208,
-      "emotet": 209,
-      "troldesh": 210,
-      "kovter": 211,
-      "boleto": 212,
-      "hancitor": 213,
-      "remcos": 214,
-      "trickbot": 215,
-      "rig": 216,
-      "tofsee": 217,
-      "neutrino": 218,
-      "icedid": 219,
-      "dreambot": 220,
-      "miuref": 221,
-      "crypt": 222,
-      "cerber": 223,
-      "unclassified": 224,
-      "sigma": 225,
-      "spora": 226,
-      "locky": 227,
-      "fallout": 228,
-      "banload": 229,
-      "globeimposter": 230,
-      "angler": 231,
-      "ursnif": 232,
-      "?": 233,
-      "[PAD]": 234
-    }
-  }
-}
+{"version":"1.0","truncation":{"max_length":160,"strategy":"LongestFirst","stride":0},"padding":{"strategy":{"Fixed":160},"direction":"Right","pad_to_multiple_of":null,"pad_id":234,"pad_type_id":0,"pad_token":"[PAD]"},"added_tokens":[{"id":1,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":3,"special":true,"content":"[end]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":234,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":235,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":236,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":null,"pre_tokenizer":null,"post_processor":null,"decoder":null,"model":{"type":"WordPiece","unk_token":"[UNK]","continuing_subword_prefix":"##","max_input_chars_per_word":100,"vocab":{"":0,"[UNK]":1,"[start]":2,"[end]":3,"malicious":4,"benign":5,"alert":6,"server_name":7,"sequence":8,"TLS":9,"cipher":10,"change_cipher_spec":11,"client_key_length":12,"c0":13,"s0":14,"c1":15,"s1":16,"c2":17,"s2":18,"c3":19,"s3":20,"c4":21,"s4":22,"c5":23,"s5":24,"c6":25,"s6":26,"c7":27,"s7":28,"c8":29,"s8":30,"c9":31,"s9":32,"c10":33,"s10":34,"c11":35,"s11":36,"c12":37,"s12":38,"c13":39,"s13":40,"c14":41,"s14":42,"c15":43,"s15":44,"c16":45,"s16":46,"l<1":47,"l:1":48,"l:2":49,"l:3":50,"l:4":51,"l:5":52,"l:6":53,"l:7":54,"l:8":55,"l:9":56,"l:10":57,"l:11":58,"l:12":59,"l:13":60,"l:14":61,"l:15":62,"l:16":63,"l:17":64,"l:18":65,"l:19":66,"l:20":67,"l>20":68,"l>10":69,"a":70,"b":71,"c":72,"d":73,"e":74,"f":75,"g":76,"h":77,"i":78,"j":79,"k":80,"l":81,"m":82,"n":83,"o":84,"p":85,"q":86,"r":87,"s":88,"t":89,"u":90,"v":91,"w":92,"x":93,"y":94,"z":95,"0":96,"1":97,"2":98,"3":99,"4":100,"5":101,"6":102,"7":103,"8":104,"9":105,".":106,"-":107,"SSLv2":108,"SSLv3":109,"TLS1.0":110,"TLS1.1":111,"TLS1.2":112,"TLS1.3":113,"TLS1.3-d18":114,"TLS1.3-d19":115,"TLS_RSA_WITH_RC4_128_MD5":116,"TLS_RSA_WITH_RC4_128_SHA":117,"TLS_RSA_WITH_3DES_EDE_CBC_SHA":118,"TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA":119,"TLS_DHE_RSA_WITH_DES_CBC_SHA":120,"TLS_RSA_WITH_AES_128_CBC_SHA":121,"TLS_DHE_DSS_WITH_AES_128_CBC_SHA":122,"TLS_DHE_RSA_WITH_AES_128_CBC_SHA":123,"TLS_RSA_WITH_AES_256_CBC_SHA":124,"TLS_DHE_DSS_WITH_AES_256_CBC_SHA":125,"TLS_DHE_RSA_WITH_AES_256_CBC_SHA":126,"TLS_RSA_WITH_AES_128_CBC_SHA256":127,"TLS_RSA_WITH_AES_256_CBC_SHA256":128,"TLS_DHE_DSS_WITH_AES_128_CBC_SHA256":129,"TLS_RSA_WITH_CAMELLIA_128_CBC_SHA":130,"TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA":131,"TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA":132,"SSL_RSA_EXPORT1024_WITH_RC4_56_SHA":133,"TLS_DHE_RSA_WITH_AES_128_CBC_SHA256":134,"TLS_DHE_DSS_WITH_AES_256_CBC_SHA256":135,"TLS_DHE_RSA_WITH_AES_256_CBC_SHA256":136,"TLS_RSA_WITH_CAMELLIA_256_CBC_SHA":137,"TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA":138,"TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA":139,"TLS_RSA_WITH_SEED_CBC_SHA":140,"TLS_RSA_WITH_AES_128_GCM_SHA256":141,"TLS_RSA_WITH_AES_256_GCM_SHA384":142,"TLS_DHE_RSA_WITH_AES_128_GCM_SHA256":143,"TLS_DHE_RSA_WITH_AES_256_GCM_SHA384":144,"TLS_DHE_DSS_WITH_AES_128_GCM_SHA256":145,"TLS_DHE_DSS_WITH_AES_256_GCM_SHA384":146,"TLS_DHE_PSK_WITH_AES_128_GCM_SHA256":147,"TLS_DHE_PSK_WITH_AES_256_GCM_SHA384":148,"TLS_AES_128_GCM_SHA256":149,"TLS_AES_256_GCM_SHA384":150,"TLS_CHACHA20_POLY1305_SHA256":151,"TLS_AES_128_CCM_SHA256":152,"TLS_ECDHE_ECDSA_WITH_RC4_128_SHA":153,"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA":154,"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA":155,"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA":156,"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA":157,"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256":158,"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384":159,"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256":160,"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384":161,"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256":162,"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384":163,"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256":164,"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384":165,"TLS_DHE_RSA_WITH_AES_128_CCM":166,"TLS_DHE_RSA_WITH_AES_256_CCM":167,"TLS_DHE_RSA_WITH_AES_128_CCM_8":168,"TLS_DHE_RSA_WITH_AES_256_CCM_8":169,"TLS_DHE_PSK_WITH_AES_128_CCM":170,"TLS_DHE_PSK_WITH_AES_256_CCM":171,"TLS_ECDHE_ECDSA_WITH_AES_128_CCM":172,"TLS_ECDHE_ECDSA_WITH_AES_256_CCM":173,"TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8":174,"TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8":175,"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256":176,"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256":177,"TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256":178,"TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256":179,"TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256":180,"TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256":181,"TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384":182,"TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256":183,"FIN":184,"SYN":185,"RST":186,"PSH":187,"ACK":188,"URG":189,"ECE":190,"CWR":191,"ramnit":192,"crthrazy":193,"nymaim":194,"bunitu":195,"zeus":196,"azorult":197,"parite":198,"vawtrak":199,"reposfxg":200,"zeus-panda":201,"gandcrab":202,"bankerx":203,"gootkit":204,"dridex":205,"upatre":206,"qakbot":207,"chthonic":208,"emotet":209,"troldesh":210,"kovter":211,"boleto":212,"hancitor":213,"remcos":214,"trickbot":215,"rig":216,"tofsee":217,"neutrino":218,"icedid":219,"dreambot":220,"miuref":221,"crypt":222,"cerber":223,"unclassified":224,"sigma":225,"spora":226,"locky":227,"fallout":228,"banload":229,"globeimposter":230,"angler":231,"ursnif":232,"?":233,"[PAD]":234}}}

tokenizer_config.json

@@ -1 +1 @@
-{"max_len": 256, "tokenizer_class": "PreTrainedTokenizerFast"}
+{"max_len": 256, "special_tokens_map_file": "/root/.cache/huggingface/transformers/85a76eea59fe40ae80bc50b05c4fe93e7547727086c4a19787726e35a451f9fd.45ed21ffc69cb3eceab51050529cfc4e1b82b5f17027779bf75c6eacc17a5079", "name_or_path": "./out/ZeroQuant/W8A8_quantization/best", "tokenizer_class": "PreTrainedTokenizerFast"}