add tokenizer

runs/May06_13-43-21_hydcorp-esxi09/1651824812.4748127/events.out.tfevents.1651824812.hydcorp-esxi09.1532484.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b40fe33ea12fdafe552cef248c0fd8d3e9afd4a8caf8ac9159c92c60ed00705a
+size 4792

runs/May06_13-43-21_hydcorp-esxi09/events.out.tfevents.1651824812.hydcorp-esxi09.1532484.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6511f4346d3403d065fbb8fcd459159360453c7d6c0d918a9590d42ff181c58b
+size 4757

vocab.json

@@ -1 +1 @@
-{"j": 0, "9": 1, "v": 2, "3": 3, "8": 4, "4": 5, "h": 7, "n": 8, "e": 9, "1": 10, "x": 11, "f": 12, "o": 13, "y": 14, "p": 15, "5": 16, "0": 17, "b": 18, "l": 19, "d": 20, "6": 21, "c": 22, "2": 23, "z": 24, "q": 25, "a": 26, "m": 27, "u": 28, "'": 29, "i": 30, "g": 31, "w": 32, "t": 33, "k": 34, "r": 35, "s": 36, "|": 6, "[UNK]": 37, "[PAD]": 38}
+{"'": 0, "q": 1, "c": 2, "t": 3, "r": 4, "d": 5, "b": 6, "p": 7, "8": 8, "n": 10, "2": 11, "u": 12, "o": 13, "x": 14, "a": 15, "0": 16, "6": 17, "5": 18, "m": 19, "h": 20, "i": 21, "f": 22, "z": 23, "9": 24, "4": 25, "y": 26, "g": 27, "3": 28, "s": 29, "l": 30, "k": 31, "e": 32, "w": 33, "1": 34, "j": 35, "v": 36, "|": 9, "[UNK]": 37, "[PAD]": 38}