add tokenizer

runs/Jun14_17-26-39_container-c6d2118f3c-03acbaae/1655199091.7823005/events.out.tfevents.1655199091.container-c6d2118f3c-03acbaae.299.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac71e1fe9b5dc9c5ea1b9db8a6990eb9876b2193fb90139bbffc568b66c7e71b
+size 4898

runs/Jun14_17-26-39_container-c6d2118f3c-03acbaae/events.out.tfevents.1655199091.container-c6d2118f3c-03acbaae.299.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e43f050f5c4d63d08e4bd316fe73226d5e8f0696bdb8f96eb0e08d621193cf89
+size 5228

vocab.json

@@ -1 +1 @@
-{"a": 0, "v": 1, "x": 2, "c": 3, "r": 5, "z": 6, "t": 7, "j": 8, "f": 9, "g": 10, "h": 11, "o": 12, "m": 13, "i": 14, "p": 15, "b": 17, "y": 18, "l": 19, "u": 20, "w": 21, "q": 22, "e": 23, "n": 24, "k": 25, "d": 26, "s": 27, "[UNK]": 28, "[PAD]": 29, "|": 16}
+{"q": 0, "n": 1, "s": 2, "i": 3, "e": 4, "h": 6, "d": 7, "v": 9, "x": 10, "f": 11, "m": 12, "g": 13, "p": 14, "a": 15, "y": 16, "r": 17, "u": 18, "z": 19, "o": 20, "l": 21, "t": 22, "k": 23, "c": 24, "b": 25, "w": 26, "j": 27, "[UNK]": 28, "[PAD]": 29, "|": 5}