add tokenizer

vocab.json

@@ -1 +1 @@
-{"g": 0, "0": 1, "3": 2, "u": 3, "a": 4, "6": 5, "n": 7, "%": 8, "/": 9, "s": 10, "2": 11, "o": 12, "5": 13, "k": 14, "x": 15, "y": 16, "v": 17, "p": 18, "f": 19, "r": 20, "w": 21, "_": 22, "8": 23, "4": 24, "z": 25, "c": 26, "q": 27, "t": 28, "d": 29, "h": 30, "1": 31, "m": 32, "l": 33, "'": 34, "e": 35, "i": 36, "b": 37, "j": 38, "#": 39, "|": 6, "[UNK]": 40, "[PAD]": 41}
+{"#": 0, "o": 1, "4": 2, "/": 3, "e": 5, "a": 6, "u": 7, "'": 8, "y": 9, "j": 10, "_": 11, "8": 12, "n": 13, "v": 14, "r": 15, "3": 16, "c": 17, "k": 18, "0": 19, "t": 20, "q": 21, "%": 22, "z": 23, "s": 24, "l": 25, "1": 26, "m": 27, "h": 28, "f": 29, "d": 30, "6": 31, "b": 32, "g": 33, "x": 34, "2": 35, "i": 36, "w": 37, "5": 38, "p": 39, "|": 4, "[UNK]": 40, "[PAD]": 41}