File size: 2,365 Bytes
0b32ad6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
"""
Basic G2P
Authors:
* Heng-Jui Chang 2022
"""
import logging
from collections import defaultdict
from typing import Dict, List, Tuple
from s3prl.util.download import _urls_to_filepaths
DEFAULT_LEXICON_URL = [
"https://huggingface.co/datasets/s3prl/g2p/raw/main/lexicon/librispeech-lexicon-200k-g2p.txt",
"https://huggingface.co/datasets/s3prl/g2p/raw/main/lexicon/librispeech-lexicon-allothers-g2p.txt",
]
__all__ = ["G2P"]
def parse_lexicon(line: str) -> Tuple[str, List[str]]:
line.replace("\t", " ")
word, *phonemes = line.split()
return word, phonemes
def read_lexicon_files(file_list: List[str]) -> Dict[str, List[str]]:
w2p_dict = defaultdict(list)
for file in file_list:
with open(file, "r") as fp:
lines = [line.strip() for line in fp]
for line in lines:
word, phonemes = parse_lexicon(line)
w2p_dict[word].append(phonemes)
w2p = {}
for word, phonemes_all in w2p_dict.items():
if len(phonemes_all) > 1:
logging.info(f"{len(phonemes_all)} phoneme sequences found for {word}.")
for i, phonemes in enumerate(phonemes_all):
logging.info(f"{i}. {phonemes}")
w2p[word] = phonemes_all[0]
logging.info("Taking the first phoneme sequences for a deterministic behavior.")
return w2p
class G2P:
"""Grapheme-to-phoneme
Args:
file_list (List[str], optional): List of lexicon files. Defaults to None.
allow_unk (bool): If false, raise Error when a word can not be recognized by this basic G2P
"""
def __init__(self, file_list: List[str] = None, allow_unk: bool = False):
self.allow_unk = allow_unk
if file_list is None:
file_list = _urls_to_filepaths(*DEFAULT_LEXICON_URL)
self.word2phone = read_lexicon_files(file_list)
def encode(self, text: str) -> str:
"""Converts grapheme-based sentences to phonemes
Args:
text (str): Sentence
Returns:
str: Phonemized sentence
"""
word_list = text.strip().upper().split(" ")
phonemes = []
for word in word_list:
if not self.allow_unk:
assert word in self.word2phone
phonemes += self.word2phone.get(word, ["<UNK>"])
return " ".join(phonemes)
|