PrajwalS commited on
Commit
9b05765
·
1 Parent(s): 4174c4b

add tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer_config.json +1 -1
  2. vocab.json +1 -1
tokenizer_config.json CHANGED
@@ -1 +1 @@
1
- {"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
 
1
+ {"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
vocab.json CHANGED
@@ -1 +1 @@
1
- {"y": 0, "r": 1, "v": 2, "t": 3, "%": 4, "3": 5, "e": 6, "o": 7, "z": 8, "x": 9, "5": 10, "n": 11, "q": 12, "w": 13, "i": 14, "6": 15, "m": 16, "2": 17, "'": 18, "b": 19, "_": 20, "/": 21, "h": 22, "k": 23, "g": 24, "l": 25, "u": 26, "a": 27, "f": 28, "j": 29, "s": 31, "#": 32, "d": 33, "1": 34, "p": 35, "c": 36, "8": 37, "0": 38, "4": 39, "|": 30, "[UNK]": 40, "[PAD]": 41}
 
1
+ {"o": 0, "d": 1, "a": 2, "e": 3, "h": 4, "r": 6, "q": 7, "n": 8, "i": 9, "p": 10, "'": 11, "v": 12, "y": 13, "b": 14, "m": 15, "x": 16, "z": 17, "u": 18, "c": 19, "w": 20, "l": 21, "s": 22, "j": 23, "t": 24, "k": 25, "g": 26, "f": 27, "|": 5, "[UNK]": 28, "[PAD]": 29}