add tokenizer

runs/Jun14_10-28-38_container-cab011a09e-b25714a4/1655173907.3354673/events.out.tfevents.1655173907.container-cab011a09e-b25714a4.6243.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8915a6450f90b889c2cb8ba6e0dd16a34932117a009b74ca618ab8fa76dd12c2
+size 4898

runs/Jun14_10-28-38_container-cab011a09e-b25714a4/events.out.tfevents.1655173907.container-cab011a09e-b25714a4.6243.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:133e2699e3375af518c2f0bc6b7d2109889dc892ef24810a75a38c8e3d7748bf
+size 5228

runs/Jun14_10-45-27_container-cab011a09e-b25714a4/1655174902.4722204/events.out.tfevents.1655174902.container-cab011a09e-b25714a4.6243.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef54d340c9bccbe76ed20aad986c94f019adeeb6b9e28325c16bc55062e69105
+size 4898

runs/Jun14_10-45-27_container-cab011a09e-b25714a4/events.out.tfevents.1655174902.container-cab011a09e-b25714a4.6243.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e37c22fb7bcc2ed14e7fed1afe135d59718d4729179771f454b52a204bcc48b
+size 5228

vocab.json

@@ -1 +1 @@
-{"n": 0, "s": 1, "h": 2, "j": 3, "m": 4, "e": 5, "y": 6, "o": 7, "i": 8, "g": 9, "d": 10, "b": 11, "v": 12, "u": 13, "t": 14, "w": 15, "k": 16, "l": 17, "c": 18, "p": 19, "f": 20, "q": 21, "x": 22, "a": 23, "r": 26, "z": 27, "[UNK]": 28, "[PAD]": 29, "|": 24}
+{"i": 0, "g": 1, "e": 2, "k": 4, "b": 5, "r": 6, "d": 7, "w": 9, "v": 10, "n": 11, "x": 12, "o": 13, "z": 14, "s": 15, "a": 16, "y": 17, "l": 18, "t": 19, "m": 20, "c": 21, "u": 22, "h": 23, "p": 24, "q": 25, "j": 26, "f": 27, "[UNK]": 28, "[PAD]": 29, "|": 3}