add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 67, "</s>": 68}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -0,0 +1 @@
+{"!": 1, ",": 2, "-": 3, ".": 4, ":": 5, ";": 6, "?": 7, "A": 8, "B": 9, "C": 10, "D": 11, "E": 12, "F": 13, "G": 14, "H": 15, "I": 16, "J": 17, "K": 18, "L": 19, "M": 20, "N": 21, "O": 22, "P": 23, "R": 24, "S": 25, "T": 26, "U": 27, "V": 28, "W": 29, "X": 30, "Z": 31, "a": 32, "b": 33, "c": 34, "d": 35, "e": 36, "f": 37, "g": 38, "h": 39, "i": 40, "j": 41, "k": 42, "l": 43, "m": 44, "n": 45, "o": 46, "p": 47, "r": 48, "s": 49, "t": 50, "u": 51, "v": 52, "w": 53, "y": 54, "z": 55, "«": 56, "»": 57, "Č": 58, "č": 59, "Š": 60, "š": 61, "Ž": 62, "ž": 63, "–": 64, "|": 0, "[UNK]": 65, "[PAD]": 66}