add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

vocab.json

@@ -1 +1 @@
-{"l": 0, "j": 1, "0": 2, "y": 3, "t": 4, "3": 5, "d": 6, "c": 7, "2": 9, "g": 10, "r": 11, "9": 12, "5": 13, "b": 14, "u": 15, "p": 16, "8": 17, "o": 18, "1": 19, "z": 20, "n": 21, "7": 22, "h": 23, "e": 24, "m": 25, "k": 26, "s": 27, "f": 28, "x": 29, "v": 30, "q": 31, "w": 32, "i": 33, "4": 34, "6": 35, "a": 36, "|": 8, "[UNK]": 37, "[PAD]": 38}
+{"1": 0, "m": 1, "u": 2, "w": 3, "0": 4, "a": 5, "h": 6, "v": 7, "e": 8, "r": 9, "4": 10, "y": 12, "n": 13, "c": 14, "2": 15, "p": 16, "k": 17, "g": 18, "8": 19, "9": 20, "3": 21, "6": 22, "b": 23, "l": 24, "i": 25, "o": 26, "d": 27, "5": 28, "q": 29, "z": 30, "s": 31, "t": 32, "x": 33, "7": 34, "f": 35, "j": 36, "|": 11, "[UNK]": 37, "[PAD]": 38}