Commit
·
9a0117b
1
Parent(s):
e90de9f
add tokenizer
Browse files- special_tokens_map.json +1 -0
- tokenizer_config.json +1 -0
- vocab.json +1 -0
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
|
vocab.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"ஜ": 0, "ோ": 1, "ஒ": 2, "க": 3, "'": 4, "ீ": 5, "ப": 6, "்": 7, "ச": 8, "ஈ": 9, "ூ": 10, "ே": 11, "ஔ": 12, "உ": 13, "ஷ": 14, "ை": 15, "ய": 16, "அ": 17, "ஏ": 18, "ஸ": 19, "எ": 20, "ண": 21, "ல": 22, "ஐ": 23, "ஃ": 24, "ு": 25, "ஹ": 26, "ஓ": 27, "ஆ": 28, "ம": 29, "ஞ": 30, "வ": 31, "ற": 32, "இ": 33, "ழ": 34, "ன": 35, "ெ": 36, "ௌ": 37, "ா": 38, "ங": 39, "ி": 40, "ஊ": 42, "ந": 43, "ள": 44, "ர": 45, "த": 46, "’": 47, "ட": 48, "ொ": 49, "|": 41, "[UNK]": 50, "[PAD]": 51}
|