add tokenizer

special_tokens_map.json

@@ -1 +1,6 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -1 +1,10 @@
-{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
+{
+  "bos_token": "<s>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "pad_token": "[PAD]",
+  "replace_word_delimiter_char": " ",
+  "tokenizer_class": "Wav2Vec2CTCTokenizer",
+  "unk_token": "[UNK]",
+  "word_delimiter_token": "|"
+}

vocab.json

@@ -1 +1,53 @@
-{"ㅣ": 0, "ㅂ": 1, "ㅇ": 2, "ㅅ": 3, "ㅡ": 4, "ㅑ": 5, "ㅔ": 6, "ㄻ": 7, "ㅁ": 8, "ㅜ": 9, "ㅞ": 10, "ㅉ": 11, "ㄲ": 12, "ㅆ": 13, "ㅀ": 14, "ㅛ": 15, "ㅘ": 16, "ㅢ": 17, "ㅊ": 18, "ㅋ": 19, "ㅗ": 20, "ㄸ": 21, "ㅃ": 22, "ㄵ": 23, "ㅐ": 24, "ㄴ": 25, "ㅎ": 26, "ㅍ": 27, "ㅖ": 28, "ㄳ": 29, "ㅠ": 30, "ㄶ": 31, "ㅟ": 32, "ㅄ": 33, "ㅝ": 34, "ㅈ": 35, "ㄹ": 36, "ㅏ": 37, "ㅚ": 38, "ㅒ": 39, "ㄷ": 40, "ㅓ": 41, "ㅙ": 42, "ㅌ": 43, "ㄼ": 44, "ㄱ": 45, "ㅕ": 47, "ㄺ": 48, "|": 46, "[UNK]": 49, "[PAD]": 50}
+{
+  "[PAD]": 50,
+  "[UNK]": 49,
+  "|": 13,
+  "ㄱ": 32,
+  "ㄲ": 18,
+  "ㄳ": 48,
+  "ㄴ": 0,
+  "ㄵ": 2,
+  "ㄶ": 41,
+  "ㄷ": 43,
+  "ㄸ": 24,
+  "ㄹ": 27,
+  "ㄺ": 7,
+  "ㄻ": 38,
+  "ㄼ": 47,
+  "ㅀ": 19,
+  "ㅁ": 12,
+  "ㅂ": 33,
+  "ㅃ": 23,
+  "ㅄ": 46,
+  "ㅅ": 42,
+  "ㅆ": 29,
+  "ㅇ": 15,
+  "ㅈ": 5,
+  "ㅉ": 16,
+  "ㅊ": 31,
+  "ㅋ": 26,
+  "ㅌ": 1,
+  "ㅍ": 9,
+  "ㅎ": 25,
+  "ㅏ": 28,
+  "ㅐ": 35,
+  "ㅑ": 37,
+  "ㅒ": 22,
+  "ㅓ": 39,
+  "ㅔ": 3,
+  "ㅕ": 44,
+  "ㅖ": 6,
+  "ㅗ": 20,
+  "ㅘ": 17,
+  "ㅙ": 34,
+  "ㅚ": 40,
+  "ㅛ": 11,
+  "ㅜ": 21,
+  "ㅝ": 10,
+  "ㅞ": 8,
+  "ㅟ": 45,
+  "ㅠ": 30,
+  "ㅡ": 36,
+  "ㅢ": 14,
+  "ㅣ": 4
+}