add tokenizer

runs/May09_10-17-45_hydcorp-esxi09/1652071746.4956002/events.out.tfevents.1652071746.hydcorp-esxi09.1922704.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b14c54eba9d72123e37ab8a08dce39897484b9f0b484bf3a68d8af8c57966e8
+size 4482

runs/May09_10-17-45_hydcorp-esxi09/events.out.tfevents.1652071746.hydcorp-esxi09.1922704.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ab5a15827446ca4b34ee86c767a9ad2bb3a2e0b991474f8d3ee7b0168ea2526
+size 4144

vocab.json

@@ -1 +1 @@
-{"k": 0, "c": 1, "'": 2, "&": 3, "a": 4, "y": 5, "i": 6, "_": 7, "n": 8, "x": 9, "q": 10, "*": 11, "™": 12, "s": 13, "b": 14, "w": 15, "€": 16, "®": 17, "d": 18, "p": 19, "v": 20, "â": 21, "u": 22, "+": 23, "m": 24, "e": 25, "r": 26, "j": 27, "o": 29, ")": 30, "/": 31, "g": 32, "#": 33, "t": 34, "h": 35, "z": 36, "@": 37, "f": 38, "(": 39, "l": 40, "%": 41, "|": 28, "[UNK]": 42, "[PAD]": 43}
+{"f": 0, "v": 1, "k": 2, "t": 3, "l": 4, "b": 5, "'": 6, "h": 7, "x": 8, "w": 9, "m": 10, "e": 11, "€": 12, "d": 13, "/": 14, "s": 15, "z": 17, "p": 18, "i": 19, "y": 20, "q": 21, "a": 22, "â": 23, "r": 24, "o": 25, "j": 26, "+": 27, "c": 28, "g": 29, "n": 30, "™": 31, "®": 32, "u": 33, "|": 16, "[UNK]": 34, "[PAD]": 35}