add tokenizer

runs/Jun14_11-02-07_container-cab011a09e-b25714a4/1655175913.8431895/events.out.tfevents.1655175913.container-cab011a09e-b25714a4.33266.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3012a6815c4acd03cec6d72ca6c430fd6ecb426a39cd7614299d9393d8d6a2fa
+size 4898

runs/Jun14_11-02-07_container-cab011a09e-b25714a4/events.out.tfevents.1655175913.container-cab011a09e-b25714a4.33266.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1dc85bf5891417bb3afc1bee1f7887e249f1b7017ebf7510f937649a6a78bdd
+size 5228

runs/Jun14_11-30-27_container-cab011a09e-b25714a4/1655177604.1490252/events.out.tfevents.1655177604.container-cab011a09e-b25714a4.33266.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:196a2c91134de476fd3a967fba3c9e411891af317e9918ed80867522e451f629
+size 4898

runs/Jun14_11-30-27_container-cab011a09e-b25714a4/events.out.tfevents.1655177604.container-cab011a09e-b25714a4.33266.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f0f8e9d21393dd5058650b85fb7888bbb3307dc0feadd73f03adcbf90b0bb74
+size 5228

vocab.json

@@ -1 +1 @@
-{"i": 0, "g": 1, "e": 2, "k": 4, "b": 5, "r": 6, "d": 7, "w": 9, "v": 10, "n": 11, "x": 12, "o": 13, "z": 14, "s": 15, "a": 16, "y": 17, "l": 18, "t": 19, "m": 20, "c": 21, "u": 22, "h": 23, "p": 24, "q": 25, "j": 26, "f": 27, "[UNK]": 28, "[PAD]": 29, "|": 3}
+{"i": 0, "c": 1, "y": 3, "s": 4, "t": 5, "a": 6, "o": 7, "q": 8, "f": 9, "p": 10, "k": 11, "b": 12, "g": 13, "l": 14, "e": 15, "h": 16, "u": 18, "j": 19, "r": 20, "v": 21, "x": 22, "w": 23, "d": 24, "n": 25, "m": 26, "z": 27, "[UNK]": 28, "[PAD]": 29, "|": 17}