mgtoxd commited on
Commit
7c57931
·
1 Parent(s): aab1fbb

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"c": 0, "h": 1, "y": 2, "v": 3, "s": 4, "k": 5, "e": 7, "u": 8, "z": 9, "r": 10, "g": 11, "x": 12, "i": 14, "p": 15, "m": 16, "f": 17, "n": 18, "d": 19, "w": 21, "j": 22, "l": 23, "o": 24, "a": 25, "q": 26, "t": 27, "b": 28, "<period>": 30, "<comma>": 31, "|": 6}
 
1
+ {"j": 0, "s": 1, "i": 2, "k": 3, "e": 4, "t": 5, "u": 6, "o": 7, "l": 8, "n": 9, "c": 10, "p": 11, "v": 12, "g": 13, "q": 15, "z": 16, "w": 17, "b": 18, "h": 19, "f": 21, "y": 22, "a": 23, "x": 24, "r": 25, "d": 26, "m": 27, "<period>": 28, "<comma>": 29, "|": 20}