add tokenizer
Browse files- vocab.json +1 -1
vocab.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"
|
|
|
1 |
+
{"j": 0, "s": 1, "i": 2, "k": 3, "e": 4, "t": 5, "u": 6, "o": 7, "l": 8, "n": 9, "c": 10, "p": 11, "v": 12, "g": 13, "q": 15, "z": 16, "w": 17, "b": 18, "h": 19, "f": 21, "y": 22, "a": 23, "x": 24, "r": 25, "d": 26, "m": 27, "<period>": 28, "<comma>": 29, "|": 20}
|