add tokenizer

runs/May09_10-23-44_hydcorp-esxi09/1652072105.4265003/events.out.tfevents.1652072105.hydcorp-esxi09.1925854.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37c6e26ec071e6798a5c4f2a50710278561b2cd3b31467db0911eee6eaddf3a0
+size 4482

runs/May09_10-23-44_hydcorp-esxi09/events.out.tfevents.1652072105.hydcorp-esxi09.1925854.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1307ea93fc563f9063c0d515f2028b33afcbbb9839fc47a15d8b9fa6a00f367a
+size 4144

vocab.json

@@ -1 +1 @@
-{"f": 0, "v": 1, "k": 2, "t": 3, "l": 4, "b": 5, "'": 6, "h": 7, "x": 8, "w": 9, "m": 10, "e": 11, "€": 12, "d": 13, "/": 14, "s": 15, "z": 17, "p": 18, "i": 19, "y": 20, "q": 21, "a": 22, "â": 23, "r": 24, "o": 25, "j": 26, "+": 27, "c": 28, "g": 29, "n": 30, "™": 31, "®": 32, "u": 33, "|": 16, "[UNK]": 34, "[PAD]": 35}
+{"n": 0, "z": 1, "m": 2, "x": 3, "u": 4, "k": 5, "r": 6, "i": 7, "®": 8, "s": 9, "c": 10, "a": 11, "â": 12, "q": 13, "+": 14, "w": 15, "o": 16, "b": 17, "€": 18, "j": 19, "y": 20, "h": 21, "t": 22, "l": 23, "p": 25, "'": 26, "d": 27, "/": 28, "g": 29, "™": 30, "v": 31, "f": 32, "e": 33, "|": 24, "[UNK]": 34, "[PAD]": 35}