Delete text

text/__init__.py

@@ -1,15 +0,0 @@
-from text.symbols import *
-
-
-_symbol_to_id = {s: i for i, s in enumerate(symbols)}
-
-def cleaned_text_to_sequence(cleaned_text):
-  '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
-    Args:
-      text: string to convert to a sequence
-    Returns:
-      List of integers corresponding to the symbols in the text
-  '''
-  phones = [_symbol_to_id[symbol] for symbol in cleaned_text]
-  return phones
-

text/chinese.py

@@ -1,172 +0,0 @@
-import os
-import pdb
-import re
-
-import cn2an
-from pypinyin import lazy_pinyin, Style
-
-import sys
-
-sys.path.append("/data/docker/liujing04/gpt-vits/gpt-vits-master")
-
-from text.symbols import punctuation
-from text.tone_sandhi import ToneSandhi
-
-current_file_path = os.path.dirname(__file__)
-pinyin_to_symbol_map = {
-    line.split("\t")[0]: line.strip().split("\t")[1]
-    for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
-}
-
-import jieba_fast.posseg as psg
-
-
-rep_map = {
-    "：": ",",
-    "；": ",",
-    "，": ",",
-    "。": ".",
-    "！": "!",
-    "？": "?",
-    "\n": ".",
-    "·": ",",
-    "、": ",",
-    "...": "…",
-    "$": ".",
-    "/": ",",
-    "—": "-",
-}
-
-tone_modifier = ToneSandhi()
-
-
-def replace_punctuation(text):
-    text = text.replace("嗯", "恩").replace("呣", "母")
-    pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
-
-    replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
-
-    replaced_text = re.sub(
-        r"[^\u4e00-\u9fa5" + "".join(punctuation) + r"]+", "", replaced_text
-    )
-
-    return replaced_text
-
-
-def g2p(text):
-    pattern = r"(?<=[{0}])\s*".format("".join(punctuation))
-    sentences = [i for i in re.split(pattern, text) if i.strip() != ""]
-    phones, word2ph = _g2p(sentences)
-    return phones, word2ph
-
-
-def _get_initials_finals(word):
-    initials = []
-    finals = []
-    orig_initials = lazy_pinyin(word, neutral_tone_with_five=True, style=Style.INITIALS)
-    orig_finals = lazy_pinyin(
-        word, neutral_tone_with_five=True, style=Style.FINALS_TONE3
-    )
-    for c, v in zip(orig_initials, orig_finals):
-        initials.append(c)
-        finals.append(v)
-    return initials, finals
-
-
-def _g2p(segments):
-    phones_list = []
-    word2ph = []
-    for seg in segments:
-        pinyins = []
-        # Replace all English words in the sentence
-        seg = re.sub("[a-zA-Z]+", "", seg)
-        seg_cut = psg.lcut(seg)
-        initials = []
-        finals = []
-        seg_cut = tone_modifier.pre_merge_for_modify(seg_cut)
-        for word, pos in seg_cut:
-            if pos == "eng":
-                continue
-            sub_initials, sub_finals = _get_initials_finals(word)
-            sub_finals = tone_modifier.modified_tone(word, pos, sub_finals)
-            initials.append(sub_initials)
-            finals.append(sub_finals)
-
-            # assert len(sub_initials) == len(sub_finals) == len(word)
-        initials = sum(initials, [])
-        finals = sum(finals, [])
-        #
-        for c, v in zip(initials, finals):
-            raw_pinyin = c + v
-            # NOTE: post process for pypinyin outputs
-            # we discriminate i, ii and iii
-            if c == v:
-                assert c in punctuation
-                phone = [c]
-                word2ph.append(1)
-            else:
-                v_without_tone = v[:-1]
-                tone = v[-1]
-
-                pinyin = c + v_without_tone
-                assert tone in "12345"
-
-                if c:
-                    # 多音节
-                    v_rep_map = {
-                        "uei": "ui",
-                        "iou": "iu",
-                        "uen": "un",
-                    }
-                    if v_without_tone in v_rep_map.keys():
-                        pinyin = c + v_rep_map[v_without_tone]
-                else:
-                    # 单音节
-                    pinyin_rep_map = {
-                        "ing": "ying",
-                        "i": "yi",
-                        "in": "yin",
-                        "u": "wu",
-                    }
-                    if pinyin in pinyin_rep_map.keys():
-                        pinyin = pinyin_rep_map[pinyin]
-                    else:
-                        single_rep_map = {
-                            "v": "yu",
-                            "e": "e",
-                            "i": "y",
-                            "u": "w",
-                        }
-                        if pinyin[0] in single_rep_map.keys():
-                            pinyin = single_rep_map[pinyin[0]] + pinyin[1:]
-
-                assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin)
-                new_c, new_v = pinyin_to_symbol_map[pinyin].split(" ")
-                new_v = new_v + tone
-                phone = [new_c, new_v]
-                word2ph.append(len(phone))
-
-            phones_list += phone
-    return phones_list, word2ph
-
-
-def text_normalize(text):
-    numbers = re.findall(r"\d+(?:\.?\d+)?", text)
-    for number in numbers:
-        text = text.replace(number, cn2an.an2cn(number), 1)
-    text = replace_punctuation(text)
-
-    return text
-
-
-if __name__ == "__main__":
-    text = "啊——但是《原神》是由,米哈\游自主，研发的一款全.新开放世界.冒险游戏"
-    text = "呣呣呣～就是…大人的鼹鼠党吧？"
-    text = "你好"
-    text = text_normalize(text)
-    print(g2p(text))
-
-
-# # 示例用法
-# text = "这是一个示例文本：,你好！这是一个测试..."
-# print(g2p_paddle(text))  # 输出: 这是一个示例文本你好这是一个测试

text/cleaner.py

@@ -1,55 +0,0 @@
-from text import chinese, japanese, cleaned_text_to_sequence, symbols, english
-
-language_module_map = {"zh": chinese, "ja": japanese, "en": english}
-special = [
-    ("%", "zh", "SP"),
-    ("￥", "zh", "SP2"),
-    ("^", "zh", "SP3"),
-    # ('@', 'zh', "SP4")#不搞鬼畜了，和第二版保持一致吧
-]
-
-
-def clean_text(text, language):
-    for special_s, special_l, target_symbol in special:
-        if special_s in text and language == special_l:
-            return clean_special(text, language, special_s, target_symbol)
-    language_module = language_module_map[language]
-    norm_text = language_module.text_normalize(text)
-    if language == "zh":
-        phones, word2ph = language_module.g2p(norm_text)
-        assert len(phones) == sum(word2ph)
-        assert len(norm_text) == len(word2ph)
-    else:
-        phones = language_module.g2p(norm_text)
-        word2ph = None
-
-    for ph in phones:
-        assert ph in symbols
-    return phones, word2ph, norm_text
-
-
-def clean_special(text, language, special_s, target_symbol):
-    """
-    特殊静音段sp符号处理
-    """
-    text = text.replace(special_s, ",")
-    language_module = language_module_map[language]
-    norm_text = language_module.text_normalize(text)
-    phones = language_module.g2p(norm_text)
-    new_ph = []
-    for ph in phones:
-        assert ph in symbols
-        if ph == ",":
-            new_ph.append(target_symbol)
-        else:
-            new_ph.append(ph)
-    return new_ph
-
-
-def text_to_sequence(text, language):
-    phones = clean_text(text)
-    return cleaned_text_to_sequence(phones)
-
-
-if __name__ == "__main__":
-    print(clean_text("你好%啊啊啊额、还是到付红四方。", "zh"))

text/cmudict_cache.pickle

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b9b21b20325471934ba92f2e4a5976989e7d920caa32e7a286eacb027d197949
-size 6212655

text/english.py

@@ -1,179 +0,0 @@
-import pickle
-import os
-import re
-from g2p_en import G2p
-
-from string import punctuation
-
-from text import symbols
-
-current_file_path = os.path.dirname(__file__)
-CMU_DICT_PATH = os.path.join(current_file_path, "cmudict.rep")
-CACHE_PATH = os.path.join(current_file_path, "cmudict_cache.pickle")
-_g2p = G2p()
-
-arpa = {
-    "AH0",
-    "S",
-    "AH1",
-    "EY2",
-    "AE2",
-    "EH0",
-    "OW2",
-    "UH0",
-    "NG",
-    "B",
-    "G",
-    "AY0",
-    "M",
-    "AA0",
-    "F",
-    "AO0",
-    "ER2",
-    "UH1",
-    "IY1",
-    "AH2",
-    "DH",
-    "IY0",
-    "EY1",
-    "IH0",
-    "K",
-    "N",
-    "W",
-    "IY2",
-    "T",
-    "AA1",
-    "ER1",
-    "EH2",
-    "OY0",
-    "UH2",
-    "UW1",
-    "Z",
-    "AW2",
-    "AW1",
-    "V",
-    "UW2",
-    "AA2",
-    "ER",
-    "AW0",
-    "UW0",
-    "R",
-    "OW1",
-    "EH1",
-    "ZH",
-    "AE0",
-    "IH2",
-    "IH",
-    "Y",
-    "JH",
-    "P",
-    "AY1",
-    "EY0",
-    "OY2",
-    "TH",
-    "HH",
-    "D",
-    "ER0",
-    "CH",
-    "AO1",
-    "AE1",
-    "AO2",
-    "OY1",
-    "AY2",
-    "IH1",
-    "OW0",
-    "L",
-    "SH",
-}
-
-
-def replace_phs(phs):
-    rep_map = {";": ",", ":": ",", "'": "-", '"': "-"}
-    phs_new = []
-    for ph in phs:
-        if ph in symbols:
-            phs_new.append(ph)
-        elif ph in rep_map.keys():
-            phs_new.append(rep_map[ph])
-        else:
-            print("ph not in symbols: ", ph)
-    return phs_new
-
-
-def read_dict():
-    g2p_dict = {}
-    start_line = 49
-    with open(CMU_DICT_PATH) as f:
-        line = f.readline()
-        line_index = 1
-        while line:
-            if line_index >= start_line:
-                line = line.strip()
-                word_split = line.split("  ")
-                word = word_split[0]
-
-                syllable_split = word_split[1].split(" - ")
-                g2p_dict[word] = []
-                for syllable in syllable_split:
-                    phone_split = syllable.split(" ")
-                    g2p_dict[word].append(phone_split)
-
-            line_index = line_index + 1
-            line = f.readline()
-
-    return g2p_dict
-
-
-def cache_dict(g2p_dict, file_path):
-    with open(file_path, "wb") as pickle_file:
-        pickle.dump(g2p_dict, pickle_file)
-
-
-def get_dict():
-    if os.path.exists(CACHE_PATH):
-        with open(CACHE_PATH, "rb") as pickle_file:
-            g2p_dict = pickle.load(pickle_file)
-    else:
-        g2p_dict = read_dict()
-        cache_dict(g2p_dict, CACHE_PATH)
-
-    return g2p_dict
-
-
-eng_dict = get_dict()
-
-
-def text_normalize(text):
-    # todo: eng text normalize
-    return text.replace(";", ",")
-
-
-def g2p(text):
-    phones = []
-    words = re.split(r"([,;.\-\?\!\s+])", text)
-    for w in words:
-        if w.upper() in eng_dict:
-            phns = eng_dict[w.upper()]
-            for ph in phns:
-                phones += ph
-        else:
-            phone_list = list(filter(lambda p: p != " ", _g2p(w)))
-            for ph in phone_list:
-                if ph in arpa:
-                    phones.append(ph)
-                else:
-                    phones.append(ph)
-
-    return replace_phs(phones)
-
-
-if __name__ == "__main__":
-    # print(get_dict())
-    print(g2p("hello"))
-    print(g2p("In this; paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))
-    # all_phones = set()
-    # for k, syllables in eng_dict.items():
-    #     for group in syllables:
-    #         for ph in group:
-    #             all_phones.add(ph)
-    # print(all_phones)

text/japanese.py

@@ -1,106 +0,0 @@
-# modified from https://github.com/CjangCjengh/vits/blob/main/text/japanese.py
-import re
-import sys
-
-import pyopenjtalk
-
-from text import symbols
-
-# Regular expression matching Japanese without punctuation marks:
-_japanese_characters = re.compile(
-    r"[A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]"
-)
-
-# Regular expression matching non-Japanese characters or punctuation marks:
-_japanese_marks = re.compile(
-    r"[^A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]"
-)
-
-# List of (symbol, Japanese) pairs for marks:
-_symbols_to_japanese = [(re.compile("%s" % x[0]), x[1]) for x in [("％", "パーセント")]]
-
-
-# List of (consonant, sokuon) pairs:
-_real_sokuon = [
-    (re.compile("%s" % x[0]), x[1])
-    for x in [
-        (r"Q([↑↓]*[kg])", r"k#\1"),
-        (r"Q([↑↓]*[tdjʧ])", r"t#\1"),
-        (r"Q([↑↓]*[sʃ])", r"s\1"),
-        (r"Q([↑↓]*[pb])", r"p#\1"),
-    ]
-]
-
-# List of (consonant, hatsuon) pairs:
-_real_hatsuon = [
-    (re.compile("%s" % x[0]), x[1])
-    for x in [
-        (r"N([↑↓]*[pbm])", r"m\1"),
-        (r"N([↑↓]*[ʧʥj])", r"n^\1"),
-        (r"N([↑↓]*[tdn])", r"n\1"),
-        (r"N([↑↓]*[kg])", r"ŋ\1"),
-    ]
-]
-
-
-def post_replace_ph(ph):
-    rep_map = {
-        "：": ",",
-        "；": ",",
-        "，": ",",
-        "。": ".",
-        "！": "!",
-        "？": "?",
-        "\n": ".",
-        "·": ",",
-        "、": ",",
-        "...": "…",
-    }
-    if ph in rep_map.keys():
-        ph = rep_map[ph]
-    if ph in symbols:
-        return ph
-    if ph not in symbols:
-        ph = "UNK"
-    return ph
-
-
-def symbols_to_japanese(text):
-    for regex, replacement in _symbols_to_japanese:
-        text = re.sub(regex, replacement, text)
-    return text
-
-
-def preprocess_jap(text):
-    """Reference https://r9y9.github.io/ttslearn/latest/notebooks/ch10_Recipe-Tacotron.html"""
-    text = symbols_to_japanese(text)
-    sentences = re.split(_japanese_marks, text)
-    marks = re.findall(_japanese_marks, text)
-    text = []
-    for i, sentence in enumerate(sentences):
-        if re.match(_japanese_characters, sentence):
-            p = pyopenjtalk.g2p(sentence)
-            text += p.split(" ")
-
-        if i < len(marks):
-            text += [marks[i].replace(" ", "")]
-    return text
-
-
-def text_normalize(text):
-    # todo: jap text normalize
-    return text
-
-
-def g2p(norm_text):
-    phones = preprocess_jap(norm_text)
-    phones = [post_replace_ph(i) for i in phones]
-    # todo: implement tones and word2ph
-    return phones
-
-
-if __name__ == "__main__":
-    for line in open("../../../Downloads/transcript_utf8.txt").readlines():
-        text = line.split(":")[1]
-        phones = g2p(text)
-        print(phones)

text/opencpop-strict.txt

@@ -1,429 +0,0 @@
-a	AA a
-ai	AA ai
-an	AA an
-ang	AA ang
-ao	AA ao
-ba	b a
-bai	b ai
-ban	b an
-bang	b ang
-bao	b ao
-bei	b ei
-ben	b en
-beng	b eng
-bi	b i
-bian	b ian
-biao	b iao
-bie	b ie
-bin	b in
-bing	b ing
-bo	b o
-bu	b u
-ca	c a
-cai	c ai
-can	c an
-cang	c ang
-cao	c ao
-ce	c e
-cei	c ei
-cen	c en
-ceng	c eng
-cha	ch a
-chai	ch ai
-chan	ch an
-chang	ch ang
-chao	ch ao
-che	ch e
-chen	ch en
-cheng	ch eng
-chi	ch ir
-chong	ch ong
-chou	ch ou
-chu	ch u
-chua	ch ua
-chuai	ch uai
-chuan	ch uan
-chuang	ch uang
-chui	ch ui
-chun	ch un
-chuo	ch uo
-ci	c i0
-cong	c ong
-cou	c ou
-cu	c u
-cuan	c uan
-cui	c ui
-cun	c un
-cuo	c uo
-da	d a
-dai	d ai
-dan	d an
-dang	d ang
-dao	d ao
-de	d e
-dei	d ei
-den	d en
-deng	d eng
-di	d i
-dia	d ia
-dian	d ian
-diao	d iao
-die	d ie
-ding	d ing
-diu	d iu
-dong	d ong
-dou	d ou
-du	d u
-duan	d uan
-dui	d ui
-dun	d un
-duo	d uo
-e	EE e
-ei	EE ei
-en	EE en
-eng	EE eng
-er	EE er
-fa	f a
-fan	f an
-fang	f ang
-fei	f ei
-fen	f en
-feng	f eng
-fo	f o
-fou	f ou
-fu	f u
-ga	g a
-gai	g ai
-gan	g an
-gang	g ang
-gao	g ao
-ge	g e
-gei	g ei
-gen	g en
-geng	g eng
-gong	g ong
-gou	g ou
-gu	g u
-gua	g ua
-guai	g uai
-guan	g uan
-guang	g uang
-gui	g ui
-gun	g un
-guo	g uo
-ha	h a
-hai	h ai
-han	h an
-hang	h ang
-hao	h ao
-he	h e
-hei	h ei
-hen	h en
-heng	h eng
-hong	h ong
-hou	h ou
-hu	h u
-hua	h ua
-huai	h uai
-huan	h uan
-huang	h uang
-hui	h ui
-hun	h un
-huo	h uo
-ji	j i
-jia	j ia
-jian	j ian
-jiang	j iang
-jiao	j iao
-jie	j ie
-jin	j in
-jing	j ing
-jiong	j iong
-jiu	j iu
-ju	j v
-jv	j v
-juan	j van
-jvan	j van
-jue	j ve
-jve	j ve
-jun	j vn
-jvn	j vn
-ka	k a
-kai	k ai
-kan	k an
-kang	k ang
-kao	k ao
-ke	k e
-kei	k ei
-ken	k en
-keng	k eng
-kong	k ong
-kou	k ou
-ku	k u
-kua	k ua
-kuai	k uai
-kuan	k uan
-kuang	k uang
-kui	k ui
-kun	k un
-kuo	k uo
-la	l a
-lai	l ai
-lan	l an
-lang	l ang
-lao	l ao
-le	l e
-lei	l ei
-leng	l eng
-li	l i
-lia	l ia
-lian	l ian
-liang	l iang
-liao	l iao
-lie	l ie
-lin	l in
-ling	l ing
-liu	l iu
-lo	l o
-long	l ong
-lou	l ou
-lu	l u
-luan	l uan
-lun	l un
-luo	l uo
-lv	l v
-lve	l ve
-ma	m a
-mai	m ai
-man	m an
-mang	m ang
-mao	m ao
-me	m e
-mei	m ei
-men	m en
-meng	m eng
-mi	m i
-mian	m ian
-miao	m iao
-mie	m ie
-min	m in
-ming	m ing
-miu	m iu
-mo	m o
-mou	m ou
-mu	m u
-na	n a
-nai	n ai
-nan	n an
-nang	n ang
-nao	n ao
-ne	n e
-nei	n ei
-nen	n en
-neng	n eng
-ni	n i
-nian	n ian
-niang	n iang
-niao	n iao
-nie	n ie
-nin	n in
-ning	n ing
-niu	n iu
-nong	n ong
-nou	n ou
-nu	n u
-nuan	n uan
-nun	n un
-nuo	n uo
-nv	n v
-nve	n ve
-o	OO o
-ou	OO ou
-pa	p a
-pai	p ai
-pan	p an
-pang	p ang
-pao	p ao
-pei	p ei
-pen	p en
-peng	p eng
-pi	p i
-pian	p ian
-piao	p iao
-pie	p ie
-pin	p in
-ping	p ing
-po	p o
-pou	p ou
-pu	p u
-qi	q i
-qia	q ia
-qian	q ian
-qiang	q iang
-qiao	q iao
-qie	q ie
-qin	q in
-qing	q ing
-qiong	q iong
-qiu	q iu
-qu	q v
-qv	q v
-quan	q van
-qvan	q van
-que	q ve
-qve	q ve
-qun	q vn
-qvn	q vn
-ran	r an
-rang	r ang
-rao	r ao
-re	r e
-ren	r en
-reng	r eng
-ri	r ir
-rong	r ong
-rou	r ou
-ru	r u
-rua	r ua
-ruan	r uan
-rui	r ui
-run	r un
-ruo	r uo
-sa	s a
-sai	s ai
-san	s an
-sang	s ang
-sao	s ao
-se	s e
-sen	s en
-seng	s eng
-sha	sh a
-shai	sh ai
-shan	sh an
-shang	sh ang
-shao	sh ao
-she	sh e
-shei	sh ei
-shen	sh en
-sheng	sh eng
-shi	sh ir
-shou	sh ou
-shu	sh u
-shua	sh ua
-shuai	sh uai
-shuan	sh uan
-shuang	sh uang
-shui	sh ui
-shun	sh un
-shuo	sh uo
-si	s i0
-song	s ong
-sou	s ou
-su	s u
-suan	s uan
-sui	s ui
-sun	s un
-suo	s uo
-ta	t a
-tai	t ai
-tan	t an
-tang	t ang
-tao	t ao
-te	t e
-tei	t ei
-teng	t eng
-ti	t i
-tian	t ian
-tiao	t iao
-tie	t ie
-ting	t ing
-tong	t ong
-tou	t ou
-tu	t u
-tuan	t uan
-tui	t ui
-tun	t un
-tuo	t uo
-wa	w a
-wai	w ai
-wan	w an
-wang	w ang
-wei	w ei
-wen	w en
-weng	w eng
-wo	w o
-wu	w u
-xi	x i
-xia	x ia
-xian	x ian
-xiang	x iang
-xiao	x iao
-xie	x ie
-xin	x in
-xing	x ing
-xiong	x iong
-xiu	x iu
-xu	x v
-xv	x v
-xuan	x van
-xvan	x van
-xue	x ve
-xve	x ve
-xun	x vn
-xvn	x vn
-ya	y a
-yan	y En
-yang	y ang
-yao	y ao
-ye	y E
-yi	y i
-yin	y in
-ying	y ing
-yo	y o
-yong	y ong
-you	y ou
-yu	y v
-yv	y v
-yuan	y van
-yvan	y van
-yue	y ve
-yve	y ve
-yun	y vn
-yvn	y vn
-za	z a
-zai	z ai
-zan	z an
-zang	z ang
-zao	z ao
-ze	z e
-zei	z ei
-zen	z en
-zeng	z eng
-zha	zh a
-zhai	zh ai
-zhan	zh an
-zhang	zh ang
-zhao	zh ao
-zhe	zh e
-zhei	zh ei
-zhen	zh en
-zheng	zh eng
-zhi	zh ir
-zhong	zh ong
-zhou	zh ou
-zhu	zh u
-zhua	zh ua
-zhuai	zh uai
-zhuan	zh uan
-zhuang	zh uang
-zhui	zh ui
-zhun	zh un
-zhuo	zh uo
-zi	z i0
-zong	z ong
-zou	z ou
-zu	z u
-zuan	z uan
-zui	z ui
-zun	z un
-zuo	z uo

text/symbols.py

@@ -1,397 +0,0 @@
-import os
-
-# punctuation = ['!', '?', '…', ",", ".","@"]#@是SP停顿
-punctuation = ["!", "?", "…", ",", "."]  # @是SP停顿
-punctuation.append("-")
-pu_symbols = punctuation + ["SP", "SP2", "SP3", "UNK"]
-# pu_symbols = punctuation + ["SP", 'SP2', 'SP3','SP4', "UNK"]
-pad = "_"
-
-c = [
-    "AA",
-    "EE",
-    "OO",
-    "b",
-    "c",
-    "ch",
-    "d",
-    "f",
-    "g",
-    "h",
-    "j",
-    "k",
-    "l",
-    "m",
-    "n",
-    "p",
-    "q",
-    "r",
-    "s",
-    "sh",
-    "t",
-    "w",
-    "x",
-    "y",
-    "z",
-    "zh",
-]
-v = [
-    "E1",
-    "En1",
-    "a1",
-    "ai1",
-    "an1",
-    "ang1",
-    "ao1",
-    "e1",
-    "ei1",
-    "en1",
-    "eng1",
-    "er1",
-    "i1",
-    "i01",
-    "ia1",
-    "ian1",
-    "iang1",
-    "iao1",
-    "ie1",
-    "in1",
-    "ing1",
-    "iong1",
-    "ir1",
-    "iu1",
-    "o1",
-    "ong1",
-    "ou1",
-    "u1",
-    "ua1",
-    "uai1",
-    "uan1",
-    "uang1",
-    "ui1",
-    "un1",
-    "uo1",
-    "v1",
-    "van1",
-    "ve1",
-    "vn1",
-    "E2",
-    "En2",
-    "a2",
-    "ai2",
-    "an2",
-    "ang2",
-    "ao2",
-    "e2",
-    "ei2",
-    "en2",
-    "eng2",
-    "er2",
-    "i2",
-    "i02",
-    "ia2",
-    "ian2",
-    "iang2",
-    "iao2",
-    "ie2",
-    "in2",
-    "ing2",
-    "iong2",
-    "ir2",
-    "iu2",
-    "o2",
-    "ong2",
-    "ou2",
-    "u2",
-    "ua2",
-    "uai2",
-    "uan2",
-    "uang2",
-    "ui2",
-    "un2",
-    "uo2",
-    "v2",
-    "van2",
-    "ve2",
-    "vn2",
-    "E3",
-    "En3",
-    "a3",
-    "ai3",
-    "an3",
-    "ang3",
-    "ao3",
-    "e3",
-    "ei3",
-    "en3",
-    "eng3",
-    "er3",
-    "i3",
-    "i03",
-    "ia3",
-    "ian3",
-    "iang3",
-    "iao3",
-    "ie3",
-    "in3",
-    "ing3",
-    "iong3",
-    "ir3",
-    "iu3",
-    "o3",
-    "ong3",
-    "ou3",
-    "u3",
-    "ua3",
-    "uai3",
-    "uan3",
-    "uang3",
-    "ui3",
-    "un3",
-    "uo3",
-    "v3",
-    "van3",
-    "ve3",
-    "vn3",
-    "E4",
-    "En4",
-    "a4",
-    "ai4",
-    "an4",
-    "ang4",
-    "ao4",
-    "e4",
-    "ei4",
-    "en4",
-    "eng4",
-    "er4",
-    "i4",
-    "i04",
-    "ia4",
-    "ian4",
-    "iang4",
-    "iao4",
-    "ie4",
-    "in4",
-    "ing4",
-    "iong4",
-    "ir4",
-    "iu4",
-    "o4",
-    "ong4",
-    "ou4",
-    "u4",
-    "ua4",
-    "uai4",
-    "uan4",
-    "uang4",
-    "ui4",
-    "un4",
-    "uo4",
-    "v4",
-    "van4",
-    "ve4",
-    "vn4",
-    "E5",
-    "En5",
-    "a5",
-    "ai5",
-    "an5",
-    "ang5",
-    "ao5",
-    "e5",
-    "ei5",
-    "en5",
-    "eng5",
-    "er5",
-    "i5",
-    "i05",
-    "ia5",
-    "ian5",
-    "iang5",
-    "iao5",
-    "ie5",
-    "in5",
-    "ing5",
-    "iong5",
-    "ir5",
-    "iu5",
-    "o5",
-    "ong5",
-    "ou5",
-    "u5",
-    "ua5",
-    "uai5",
-    "uan5",
-    "uang5",
-    "ui5",
-    "un5",
-    "uo5",
-    "v5",
-    "van5",
-    "ve5",
-    "vn5",
-]
-
-v_without_tone = [
-    "E",
-    "En",
-    "a",
-    "ai",
-    "an",
-    "ang",
-    "ao",
-    "e",
-    "ei",
-    "en",
-    "eng",
-    "er",
-    "i",
-    "i0",
-    "ia",
-    "ian",
-    "iang",
-    "iao",
-    "ie",
-    "in",
-    "ing",
-    "iong",
-    "ir",
-    "iu",
-    "o",
-    "ong",
-    "ou",
-    "u",
-    "ua",
-    "uai",
-    "uan",
-    "uang",
-    "ui",
-    "un",
-    "uo",
-    "v",
-    "van",
-    "ve",
-    "vn",
-]
-
-# japanese
-ja_symbols = [
-    "I",
-    "N",
-    "U",
-    "a",
-    "b",
-    "by",
-    "ch",
-    "cl",
-    "d",
-    "dy",
-    "e",
-    "f",
-    "g",
-    "gy",
-    "h",
-    "hy",
-    "i",
-    "j",
-    "k",
-    "ky",
-    "m",
-    "my",
-    "n",
-    "ny",
-    "o",
-    "p",
-    "py",
-    "r",
-    "ry",
-    "s",
-    "sh",
-    "t",
-    "ts",
-    "u",
-    "v",
-    "w",
-    "y",
-    "z",
-]
-
-arpa = {
-    "AH0",
-    "S",
-    "AH1",
-    "EY2",
-    "AE2",
-    "EH0",
-    "OW2",
-    "UH0",
-    "NG",
-    "B",
-    "G",
-    "AY0",
-    "M",
-    "AA0",
-    "F",
-    "AO0",
-    "ER2",
-    "UH1",
-    "IY1",
-    "AH2",
-    "DH",
-    "IY0",
-    "EY1",
-    "IH0",
-    "K",
-    "N",
-    "W",
-    "IY2",
-    "T",
-    "AA1",
-    "ER1",
-    "EH2",
-    "OY0",
-    "UH2",
-    "UW1",
-    "Z",
-    "AW2",
-    "AW1",
-    "V",
-    "UW2",
-    "AA2",
-    "ER",
-    "AW0",
-    "UW0",
-    "R",
-    "OW1",
-    "EH1",
-    "ZH",
-    "AE0",
-    "IH2",
-    "IH",
-    "Y",
-    "JH",
-    "P",
-    "AY1",
-    "EY0",
-    "OY2",
-    "TH",
-    "HH",
-    "D",
-    "ER0",
-    "CH",
-    "AO1",
-    "AE1",
-    "AO2",
-    "OY1",
-    "AY2",
-    "IH1",
-    "OW0",
-    "L",
-    "SH",
-}
-
-symbols = [pad] + c + v + ja_symbols + pu_symbols + list(arpa)
-symbols = sorted(set(symbols))
-if __name__ == "__main__":
-    print(len(symbols))

text/tone_sandhi.py

@@ -1,805 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from typing import List
-from typing import Tuple
-
-import jieba_fast as jieba
-from pypinyin import lazy_pinyin
-from pypinyin import Style
-
-
-class ToneSandhi:
-    def __init__(self):
-        self.must_neural_tone_words = {
-            "麻烦",
-            "麻利",
-            "鸳鸯",
-            "高粱",
-            "骨头",
-            "骆驼",
-            "马虎",
-            "首饰",
-            "馒头",
-            "馄饨",
-            "风筝",
-            "难为",
-            "队伍",
-            "阔气",
-            "闺女",
-            "门道",
-            "锄头",
-            "铺盖",
-            "铃铛",
-            "铁匠",
-            "钥匙",
-            "里脊",
-            "里头",
-            "部分",
-            "那么",
-            "道士",
-            "造化",
-            "迷糊",
-            "连累",
-            "这么",
-            "这个",
-            "运气",
-            "过去",
-            "软和",
-            "转悠",
-            "踏实",
-            "跳蚤",
-            "跟头",
-            "趔趄",
-            "财主",
-            "豆腐",
-            "讲究",
-            "记性",
-            "记号",
-            "认识",
-            "规矩",
-            "见识",
-            "裁缝",
-            "补丁",
-            "衣裳",
-            "衣服",
-            "衙门",
-            "街坊",
-            "行李",
-            "行当",
-            "蛤蟆",
-            "蘑菇",
-            "薄荷",
-            "葫芦",
-            "葡萄",
-            "萝卜",
-            "荸荠",
-            "苗条",
-            "苗头",
-            "苍蝇",
-            "芝麻",
-            "舒服",
-            "舒坦",
-            "舌头",
-            "自在",
-            "膏药",
-            "脾气",
-            "脑袋",
-            "脊梁",
-            "能耐",
-            "胳膊",
-            "胭脂",
-            "胡萝",
-            "胡琴",
-            "胡同",
-            "聪明",
-            "耽误",
-            "耽搁",
-            "耷拉",
-            "耳朵",
-            "老爷",
-            "老实",
-            "老婆",
-            "老头",
-            "老太",
-            "翻腾",
-            "罗嗦",
-            "罐头",
-            "编辑",
-            "结实",
-            "红火",
-            "累赘",
-            "糨糊",
-            "糊涂",
-            "精神",
-            "粮食",
-            "簸箕",
-            "篱笆",
-            "算计",
-            "算盘",
-            "答应",
-            "笤帚",
-            "笑语",
-            "笑话",
-            "窟窿",
-            "窝囊",
-            "窗户",
-            "稳当",
-            "稀罕",
-            "称呼",
-            "秧歌",
-            "秀气",
-            "秀才",
-            "福气",
-            "祖宗",
-            "砚台",
-            "码头",
-            "石榴",
-            "石头",
-            "石匠",
-            "知识",
-            "眼睛",
-            "眯缝",
-            "眨巴",
-            "眉毛",
-            "相声",
-            "盘算",
-            "白净",
-            "痢疾",
-            "痛快",
-            "疟疾",
-            "疙瘩",
-            "疏忽",
-            "畜生",
-            "生意",
-            "甘蔗",
-            "琵琶",
-            "琢磨",
-            "琉璃",
-            "玻璃",
-            "玫瑰",
-            "玄乎",
-            "狐狸",
-            "状元",
-            "特务",
-            "牲口",
-            "牙碜",
-            "牌楼",
-            "爽快",
-            "爱人",
-            "热闹",
-            "烧饼",
-            "烟筒",
-            "烂糊",
-            "点心",
-            "炊帚",
-            "灯笼",
-            "火候",
-            "漂亮",
-            "滑溜",
-            "溜达",
-            "温和",
-            "清楚",
-            "消息",
-            "浪头",
-            "活泼",
-            "比方",
-            "正经",
-            "欺负",
-            "模糊",
-            "槟榔",
-            "棺材",
-            "棒槌",
-            "棉花",
-            "核桃",
-            "栅栏",
-            "柴火",
-            "架势",
-            "枕头",
-            "���杷",
-            "机灵",
-            "本事",
-            "木头",
-            "木匠",
-            "朋友",
-            "月饼",
-            "月亮",
-            "暖和",
-            "明白",
-            "时候",
-            "新鲜",
-            "故事",
-            "收拾",
-            "收成",
-            "提防",
-            "挖苦",
-            "挑剔",
-            "指甲",
-            "指头",
-            "拾掇",
-            "拳头",
-            "拨弄",
-            "招牌",
-            "招呼",
-            "抬举",
-            "护士",
-            "折腾",
-            "扫帚",
-            "打量",
-            "打算",
-            "打点",
-            "打扮",
-            "打听",
-            "打发",
-            "扎实",
-            "扁担",
-            "戒指",
-            "懒得",
-            "意识",
-            "意思",
-            "情形",
-            "悟性",
-            "怪物",
-            "思量",
-            "怎么",
-            "念头",
-            "念叨",
-            "快活",
-            "忙活",
-            "志气",
-            "心思",
-            "得罪",
-            "张罗",
-            "弟兄",
-            "开通",
-            "应酬",
-            "庄稼",
-            "干事",
-            "帮手",
-            "帐篷",
-            "希罕",
-            "师父",
-            "师傅",
-            "巴结",
-            "巴掌",
-            "差事",
-            "工夫",
-            "岁数",
-            "屁股",
-            "尾巴",
-            "少爷",
-            "小气",
-            "小伙",
-            "将就",
-            "对头",
-            "对付",
-            "寡妇",
-            "家伙",
-            "客气",
-            "实在",
-            "官司",
-            "学问",
-            "学生",
-            "字号",
-            "嫁妆",
-            "媳妇",
-            "媒人",
-            "婆家",
-            "娘家",
-            "委屈",
-            "姑娘",
-            "姐夫",
-            "妯娌",
-            "妥当",
-            "妖精",
-            "奴才",
-            "女婿",
-            "头发",
-            "太阳",
-            "大爷",
-            "大方",
-            "大意",
-            "大夫",
-            "多少",
-            "多么",
-            "外甥",
-            "壮实",
-            "地道",
-            "地方",
-            "在乎",
-            "困难",
-            "嘴巴",
-            "嘱咐",
-            "嘟囔",
-            "嘀咕",
-            "喜欢",
-            "喇嘛",
-            "喇叭",
-            "商量",
-            "唾沫",
-            "哑巴",
-            "哈欠",
-            "哆嗦",
-            "咳嗽",
-            "和尚",
-            "告诉",
-            "告示",
-            "含糊",
-            "吓唬",
-            "后头",
-            "名字",
-            "名堂",
-            "合同",
-            "吆喝",
-            "叫唤",
-            "口袋",
-            "厚道",
-            "厉害",
-            "千斤",
-            "包袱",
-            "包涵",
-            "匀称",
-            "勤快",
-            "动静",
-            "动弹",
-            "功夫",
-            "力气",
-            "前头",
-            "刺猬",
-            "刺激",
-            "别扭",
-            "利落",
-            "利索",
-            "利害",
-            "分析",
-            "出息",
-            "凑合",
-            "凉快",
-            "冷战",
-            "冤枉",
-            "冒失",
-            "养活",
-            "关系",
-            "先生",
-            "兄弟",
-            "便宜",
-            "使唤",
-            "佩服",
-            "作坊",
-            "体面",
-            "位置",
-            "似的",
-            "伙计",
-            "休息",
-            "什么",
-            "人家",
-            "亲戚",
-            "亲家",
-            "交情",
-            "云彩",
-            "事情",
-            "买卖",
-            "主意",
-            "丫头",
-            "丧气",
-            "两口",
-            "东西",
-            "东家",
-            "世故",
-            "不由",
-            "不在",
-            "下水",
-            "下巴",
-            "上头",
-            "上司",
-            "丈夫",
-            "丈人",
-            "一辈",
-            "那个",
-            "菩萨",
-            "父亲",
-            "母亲",
-            "咕噜",
-            "邋遢",
-            "费用",
-            "冤家",
-            "甜头",
-            "介绍",
-            "荒唐",
-            "大人",
-            "泥鳅",
-            "幸福",
-            "熟悉",
-            "计划",
-            "扑腾",
-            "蜡烛",
-            "姥爷",
-            "照顾",
-            "喉咙",
-            "吉他",
-            "弄堂",
-            "蚂蚱",
-            "凤凰",
-            "拖沓",
-            "寒碜",
-            "糟蹋",
-            "倒腾",
-            "报复",
-            "逻辑",
-            "盘缠",
-            "喽啰",
-            "牢骚",
-            "咖喱",
-            "扫把",
-            "惦记",
-        }
-        self.must_not_neural_tone_words = {
-            "男子",
-            "女子",
-            "分子",
-            "原子",
-            "量子",
-            "莲子",
-            "石子",
-            "瓜子",
-            "电子",
-            "人人",
-            "虎虎",
-            "幺幺",
-            "干嘛",
-            "学子",
-            "哈哈",
-            "数数",
-            "袅袅",
-            "局地",
-            "以下",
-            "娃哈哈",
-            "花花草草",
-            "留得",
-            "耕地",
-            "想想",
-            "熙熙",
-            "攘攘",
-            "卵子",
-            "死死",
-            "冉冉",
-            "恳恳",
-            "佼佼",
-            "吵吵",
-            "打打",
-            "考考",
-            "整整",
-            "莘莘",
-            "落地",
-            "算子",
-            "家家户户",
-            "青青",
-        }
-        self.punc = "：，；。？！“”‘’':,;.?!"
-
-    # the meaning of jieba pos tag: https://blog.csdn.net/weixin_44174352/article/details/113731041
-    # e.g.
-    # word: "家里"
-    # pos: "s"
-    # finals: ['ia1', 'i3']
-    def _neural_sandhi(self, word: str, pos: str, finals: List[str]) -> List[str]:
-        # reduplication words for n. and v. e.g. 奶奶, 试试, 旺旺
-        for j, item in enumerate(word):
-            if (
-                j - 1 >= 0
-                and item == word[j - 1]
-                and pos[0] in {"n", "v", "a"}
-                and word not in self.must_not_neural_tone_words
-            ):
-                finals[j] = finals[j][:-1] + "5"
-        ge_idx = word.find("个")
-        if len(word) >= 1 and word[-1] in "吧呢哈啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶":
-            finals[-1] = finals[-1][:-1] + "5"
-        elif len(word) >= 1 and word[-1] in "的地得":
-            finals[-1] = finals[-1][:-1] + "5"
-        # e.g. 走了, 看着, 去过
-        elif len(word) == 1 and word in "了着过" and pos in {"ul", "uz", "ug"}:
-            finals[-1] = finals[-1][:-1] + "5"
-        elif (
-            len(word) > 1
-            and word[-1] in "们子"
-            and pos in {"r", "n"}
-            and word not in self.must_not_neural_tone_words
-        ):
-            finals[-1] = finals[-1][:-1] + "5"
-        # e.g. 桌上, 地下, 家里
-        elif len(word) > 1 and word[-1] in "上下里" and pos in {"s", "l", "f"}:
-            finals[-1] = finals[-1][:-1] + "5"
-        # e.g. 上来, 下去
-        elif len(word) > 1 and word[-1] in "来去" and word[-2] in "上下进出回过起开":
-            finals[-1] = finals[-1][:-1] + "5"
-        # 个做量词
-        elif (
-            ge_idx >= 1
-            and (word[ge_idx - 1].isnumeric() or word[ge_idx - 1] in "几有两半多各整每做是")
-        ) or word == "个":
-            finals[ge_idx] = finals[ge_idx][:-1] + "5"
-        else:
-            if (
-                word in self.must_neural_tone_words
-                or word[-2:] in self.must_neural_tone_words
-            ):
-                finals[-1] = finals[-1][:-1] + "5"
-
-        word_list = self._split_word(word)
-        finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
-        for i, word in enumerate(word_list):
-            # conventional neural in Chinese
-            if (
-                word in self.must_neural_tone_words
-                or word[-2:] in self.must_neural_tone_words
-            ):
-                finals_list[i][-1] = finals_list[i][-1][:-1] + "5"
-        finals = sum(finals_list, [])
-        return finals
-
-    def _bu_sandhi(self, word: str, finals: List[str]) -> List[str]:
-        # e.g. 看不懂
-        if len(word) == 3 and word[1] == "不":
-            finals[1] = finals[1][:-1] + "5"
-        else:
-            for i, char in enumerate(word):
-                # "不" before tone4 should be bu2, e.g. 不怕
-                if char == "不" and i + 1 < len(word) and finals[i + 1][-1] == "4":
-                    finals[i] = finals[i][:-1] + "2"
-        return finals
-
-    def _yi_sandhi(self, word: str, finals: List[str]) -> List[str]:
-        # "一" in number sequences, e.g. 一零零, 二一零
-        if word.find("一") != -1 and all(
-            [item.isnumeric() for item in word if item != "一"]
-        ):
-            return finals
-        # "一" between reduplication words shold be yi5, e.g. 看一看
-        elif len(word) == 3 and word[1] == "一" and word[0] == word[-1]:
-            finals[1] = finals[1][:-1] + "5"
-        # when "一" is ordinal word, it should be yi1
-        elif word.startswith("第一"):
-            finals[1] = finals[1][:-1] + "1"
-        else:
-            for i, char in enumerate(word):
-                if char == "一" and i + 1 < len(word):
-                    # "一" before tone4 should be yi2, e.g. 一段
-                    if finals[i + 1][-1] == "4":
-                        finals[i] = finals[i][:-1] + "2"
-                    # "一" before non-tone4 should be yi4, e.g. 一天
-                    else:
-                        # "一" 后面如果是标点，还读一声
-                        if word[i + 1] not in self.punc:
-                            finals[i] = finals[i][:-1] + "4"
-        return finals
-
-    def _split_word(self, word: str) -> List[str]:
-        word_list = jieba.cut_for_search(word)
-        word_list = sorted(word_list, key=lambda i: len(i), reverse=False)
-        first_subword = word_list[0]
-        first_begin_idx = word.find(first_subword)
-        if first_begin_idx == 0:
-            second_subword = word[len(first_subword) :]
-            new_word_list = [first_subword, second_subword]
-        else:
-            second_subword = word[: -len(first_subword)]
-            new_word_list = [second_subword, first_subword]
-        return new_word_list
-
-    def _three_sandhi(self, word: str, finals: List[str]) -> List[str]:
-        if len(word) == 2 and self._all_tone_three(finals):
-            finals[0] = finals[0][:-1] + "2"
-        elif len(word) == 3:
-            word_list = self._split_word(word)
-            if self._all_tone_three(finals):
-                #  disyllabic + monosyllabic, e.g. 蒙古/包
-                if len(word_list[0]) == 2:
-                    finals[0] = finals[0][:-1] + "2"
-                    finals[1] = finals[1][:-1] + "2"
-                #  monosyllabic + disyllabic, e.g. 纸/老虎
-                elif len(word_list[0]) == 1:
-                    finals[1] = finals[1][:-1] + "2"
-            else:
-                finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
-                if len(finals_list) == 2:
-                    for i, sub in enumerate(finals_list):
-                        # e.g. 所有/人
-                        if self._all_tone_three(sub) and len(sub) == 2:
-                            finals_list[i][0] = finals_list[i][0][:-1] + "2"
-                        # e.g. 好/喜欢
-                        elif (
-                            i == 1
-                            and not self._all_tone_three(sub)
-                            and finals_list[i][0][-1] == "3"
-                            and finals_list[0][-1][-1] == "3"
-                        ):
-                            finals_list[0][-1] = finals_list[0][-1][:-1] + "2"
-                        finals = sum(finals_list, [])
-        # split idiom into two words who's length is 2
-        elif len(word) == 4:
-            finals_list = [finals[:2], finals[2:]]
-            finals = []
-            for sub in finals_list:
-                if self._all_tone_three(sub):
-                    sub[0] = sub[0][:-1] + "2"
-                finals += sub
-
-        return finals
-
-    def _all_tone_three(self, finals: List[str]) -> bool:
-        return all(x[-1] == "3" for x in finals)
-
-    # merge "不" and the word behind it
-    # if don't merge, "不" sometimes appears alone according to jieba, which may occur sandhi error
-    def _merge_bu(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        new_seg = []
-        last_word = ""
-        for word, pos in seg:
-            if last_word == "不":
-                word = last_word + word
-            if word != "不":
-                new_seg.append((word, pos))
-            last_word = word[:]
-        if last_word == "不":
-            new_seg.append((last_word, "d"))
-            last_word = ""
-        return new_seg
-
-    # function 1: merge "一" and reduplication words in it's left and right, e.g. "听","一","听" ->"听一听"
-    # function 2: merge single  "一" and the word behind it
-    # if don't merge, "一" sometimes appears alone according to jieba, which may occur sandhi error
-    # e.g.
-    # input seg: [('听', 'v'), ('一', 'm'), ('听', 'v')]
-    # output seg: [['听一听', 'v']]
-    def _merge_yi(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        new_seg = []
-        # function 1
-        for i, (word, pos) in enumerate(seg):
-            if (
-                i - 1 >= 0
-                and word == "一"
-                and i + 1 < len(seg)
-                and seg[i - 1][0] == seg[i + 1][0]
-                and seg[i - 1][1] == "v"
-            ):
-                new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0]
-            else:
-                if (
-                    i - 2 >= 0
-                    and seg[i - 1][0] == "一"
-                    and seg[i - 2][0] == word
-                    and pos == "v"
-                ):
-                    continue
-                else:
-                    new_seg.append([word, pos])
-        seg = new_seg
-        new_seg = []
-        # function 2
-        for i, (word, pos) in enumerate(seg):
-            if new_seg and new_seg[-1][0] == "一":
-                new_seg[-1][0] = new_seg[-1][0] + word
-            else:
-                new_seg.append([word, pos])
-        return new_seg
-
-    # the first and the second words are all_tone_three
-    def _merge_continuous_three_tones(
-        self, seg: List[Tuple[str, str]]
-    ) -> List[Tuple[str, str]]:
-        new_seg = []
-        sub_finals_list = [
-            lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
-            for (word, pos) in seg
-        ]
-        assert len(sub_finals_list) == len(seg)
-        merge_last = [False] * len(seg)
-        for i, (word, pos) in enumerate(seg):
-            if (
-                i - 1 >= 0
-                and self._all_tone_three(sub_finals_list[i - 1])
-                and self._all_tone_three(sub_finals_list[i])
-                and not merge_last[i - 1]
-            ):
-                # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
-                if (
-                    not self._is_reduplication(seg[i - 1][0])
-                    and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
-                ):
-                    new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
-                    merge_last[i] = True
-                else:
-                    new_seg.append([word, pos])
-            else:
-                new_seg.append([word, pos])
-
-        return new_seg
-
-    def _is_reduplication(self, word: str) -> bool:
-        return len(word) == 2 and word[0] == word[1]
-
-    # the last char of first word and the first char of second word is tone_three
-    def _merge_continuous_three_tones_2(
-        self, seg: List[Tuple[str, str]]
-    ) -> List[Tuple[str, str]]:
-        new_seg = []
-        sub_finals_list = [
-            lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
-            for (word, pos) in seg
-        ]
-        assert len(sub_finals_list) == len(seg)
-        merge_last = [False] * len(seg)
-        for i, (word, pos) in enumerate(seg):
-            if (
-                i - 1 >= 0
-                and sub_finals_list[i - 1][-1][-1] == "3"
-                and sub_finals_list[i][0][-1] == "3"
-                and not merge_last[i - 1]
-            ):
-                # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
-                if (
-                    not self._is_reduplication(seg[i - 1][0])
-                    and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
-                ):
-                    new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
-                    merge_last[i] = True
-                else:
-                    new_seg.append([word, pos])
-            else:
-                new_seg.append([word, pos])
-        return new_seg
-
-    def _merge_er(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        new_seg = []
-        for i, (word, pos) in enumerate(seg):
-            if i - 1 >= 0 and word == "儿" and seg[i - 1][0] != "#":
-                new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
-            else:
-                new_seg.append([word, pos])
-        return new_seg
-
-    def _merge_reduplication(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        new_seg = []
-        for i, (word, pos) in enumerate(seg):
-            if new_seg and word == new_seg[-1][0]:
-                new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
-            else:
-                new_seg.append([word, pos])
-        return new_seg
-
-    def pre_merge_for_modify(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        seg = self._merge_bu(seg)
-        try:
-            seg = self._merge_yi(seg)
-        except:
-            print("_merge_yi failed")
-        seg = self._merge_reduplication(seg)
-        try:
-            seg = self._merge_continuous_three_tones(seg)
-        except:
-            print("_merge_continuous_three_tones failed")
-        try:
-            seg = self._merge_continuous_three_tones_2(seg)
-        except:
-            print("_merge_continuous_three_tones_2 failed")
-
-        seg = self._merge_er(seg)
-        return seg
-
-    def modified_tone(self, word: str, pos: str, finals: List[str]) -> List[str]:
-        finals = self._bu_sandhi(word, finals)
-        finals = self._yi_sandhi(word, finals)
-        finals = self._neural_sandhi(word, pos, finals)
-        finals = self._three_sandhi(word, finals)
-        return finals