PrajwalS commited on
Commit
ff04638
·
1 Parent(s): 768b288

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"e": 0, "g": 1, "w": 2, "o": 3, "b": 4, "r": 5, "t": 6, "a": 7, "n": 8, "q": 9, "u": 10, "i": 11, "p": 12, "k": 13, "f": 14, "m": 15, "c": 16, "y": 17, "l": 18, "h": 19, "d": 21, "x": 22, "j": 23, "s": 24, "z": 25, "v": 26, "'": 27, "|": 20, "[UNK]": 28, "[PAD]": 29}
 
1
+ {"s": 0, "y": 1, "o": 2, "i": 4, "'": 5, "q": 6, "t": 7, "n": 8, "r": 9, "l": 10, "h": 11, "w": 12, "v": 13, "u": 14, "c": 15, "z": 16, "j": 17, "e": 18, "a": 19, "m": 20, "f": 21, "d": 22, "b": 23, "k": 24, "g": 25, "x": 26, "p": 27, "|": 3, "[UNK]": 28, "[PAD]": 29}