v1: single model

app.py

@@ -2,8 +2,7 @@ import os
 import numpy as np
 import gradio as gr
 import pyopenjtalk
-from pypinyin import lazy_pinyin
-from util import preprocess_input, get_tokenizer, load_pitch_dict
+from util import preprocess_input, get_tokenizer, load_pitch_dict, get_pinyin
 
 from espnet_model_zoo.downloader import ModelDownloader
 from espnet2.fileio.read_text import read_label
@@ -46,7 +45,7 @@ def gen_song(lang, texts, durs, pitchs, spk):
     # preprocess
     if lang == "zh":
         texts = preprocess_input(texts, "")
-        text_list = lazy_pinyin(texts)
+        text_list = get_pinyin(texts)
     elif lang == "jp":
         texts = preprocess_input(texts, " ")
         text_list = texts.strip().split()
@@ -66,6 +65,8 @@ def gen_song(lang, texts, durs, pitchs, spk):
     for text in text_list:
         if text == "AP" or text == "SP":
             rev = [text]
+        elif text == "-" or text == "——":
+            rev = [text]
         else:
             rev = tokenizer(text)
             rev = [phn + f"@{lang}" for phn in rev]
@@ -79,18 +80,22 @@ def gen_song(lang, texts, durs, pitchs, spk):
     labels = []
     notes = []
     st = 0
+    pre_phn = ""
     for phns, dur, pitch in zip(sybs, dur_list, pitch_list):
+        if phns == "-" or phns == "——":
+            phns = pre_phn
         if pitch not in pitch_dict:
             return (fs, np.array([0.0])), f"Error: pitch `{pitch}` is invalid!"
         pitch = pitch_dict[pitch]
-        dur = float(dur)
         phn_list = phns.split("_")
         lyric = "".join(phn_list)
+        dur = float(dur)
         note = [st, st + dur, lyric, pitch, phns]
         st += dur
         notes.append(note)
         for phn in phn_list:
             labels.append(phn)
+        pre_phn = labels[-1]
 
     phns_str = " ".join(labels)
     batch = {
@@ -125,16 +130,18 @@ def gen_song(lang, texts, durs, pitchs, spk):
 title = "Demo of Singing Voice Synthesis in Muskits-ESPnet"
 
 description = """
-<div style="font-size: 20px;">
-  <p>This is the demo page of our toolkit <b>Muskits-ESPnet: A Comprehensive Toolkit for Singing Voice Synthesis in New Paradigm</b>.</p>
+<div style="font-size: 20px;"
+  This is the demo page of our toolkit <b>Muskits-ESPnet: A Comprehensive Toolkit for Singing Voice Synthesis in New Paradigm</b>.
   <p>Singing Voice Synthesis (SVS) takes a music score as input and generates singing vocal with the voice of a specific singer.
   Music score contains information about lyrics, as well as duration and pitch of each word in lyrics.</p>
 
-  <p>How to use:</p>
+  <h1>How to use:</h1>
   <ol>
     <li> <b>Choose language ID</b>: "zh" indicates lyrics input in Chinese, and "jp" indicates lyrics input in Japanese. </li>
     <li> <b>Input lyrics</b>:
         <ul>
+            <li> Lyrics use Chinese characters when the language is 'zh' and hiragana when the language is 'jp'. </li>
+            <li> Special characters such as 'AP' (breath), 'SP' (silence), and '-' (slur, only for 'zh') can also be used. </li>
             <li> Lyrics sequence should be separated by either a space (' ') or a newline ('\\n'), without the quotation marks. </li>
         </ul>
     </li>
@@ -154,7 +161,10 @@ description = """
     <li> <b>Click submit button</b> </li>
   </ol>
 
-  <b>Notice</b>: Values outside this range may result in suboptimal generation quality!
+  <h1>Notice:</h1>
+  <ul>
+    <li> Values outside this range may result in suboptimal generation quality! </li>
+  </ul>
 </div>
 """
 
@@ -182,9 +192,20 @@ article = """
 examples = [
     ["zh", "雨 淋 湿 了 SP 天 空 AP\n毁 的 SP 很 讲 究 AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.5 0.21\n0.3 0.12 0.12 0.25 0.5 0.48 0.34", "60 62 62 62 0 62 58 0\n58 58 0 58 58 63 0", "singer1 (male)"],
     ["zh", "雨 淋 湿 了 SP 天 空 AP\n毁 的 SP 很 讲 究 AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.5 0.21\n0.3 0.12 0.12 0.25 0.5 0.48 0.34", "C4 D4 D4 D4 rest D4 A#3 rest\nA#3 A#3 rest A#3 A#3 D#4 rest", "singer1 (male)"],
-    # ["zh", 89, "雨 淋 湿 了 SP 天 空 AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.50 0.21", "C4 D4 D4 D4 rest D4 A#3 rest", "singer2 (male)"],
-    # ["zh", 89, "雨 淋 湿 了 SP 天 空 AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.50 0.21", "C4 D4 D4 D4 rest D4 Bb3 rest", "singer3 (female)"],
+    ["zh", "雨 淋 湿 了 SP 天 空 AP\n毁 的 SP 很 讲 究 AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.5 0.21\n0.3 0.12 0.12 0.25 0.5 0.48 0.34", "C#4 D#4 D#4 D#4 rest D#4 B3 rest\nB3 B3 rest B3 B3 E4 rest", "singer1 (male)"],
+    ["zh", "你 说 你 不 SP 懂\n 为 何 在 这 时 牵 手 AP", "0.11 0.33 0.29 0.13 0.15 0.48\n0.24 0.18 0.34 0.15 0.27 0.28 0.63 0.44", "63 63 63 63 0 63\n62 62 62 63 65 63 62 0", "singer1 (male)"],
+    ["zh", "你 说 你 不 SP 懂\n 为 何 在 这 时 牵 手 AP", "0.23 0.66 0.58 0.27 0.3 0.97\n0.48 0.36 0.69 0.3 0.53 0.56 1.27 0.89", "63 63 63 63 0 63\n62 62 62 63 65 63 62 0", "singer1 (male)"],
+    ["zh", "雨 淋 湿 了 SP 天 空 AP\n毁 的 SP 很 讲 究 AP\n你 说 你 不 SP 懂\n 为 何 在 这 时 牵 手 AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.5 0.21\n0.3 0.12 0.12 0.25 0.5 0.48 0.34\n0.11 0.33 0.29 0.13 0.15 0.48\n0.24 0.18 0.34 0.15 0.27 0.28 0.63 0.44", "60 62 62 62 0 62 58 0\n58 58 0 58 58 63 0\n63 63 63 63 0 63\n62 62 62 63 65 63 62 0", "singer1 (male)"],
+    ["zh", "修 炼 爱 情 的 心 酸 SP AP", "0.42 0.21 0.19 0.28 0.22 0.33 1.53 0.1 0.29", "68 70 68 66 63 68 68 0 0", "singer2 (female)"],
+    ["zh", "学 会 放 好 以 前 的 渴 望 SP AP", "0.3 0.22 0.29 0.27 0.25 0.44 0.54 0.29 1.03 0.08 0.39", "68 70 68 66 61 68 68 65 66 0 0", "singer2 (female)"],
+    ["zh", "SP 你 看 着 车 窗 - SP", " 0.41 0.96 0.7 0.64 1.12 1.14 1.04 0.29", "0 60 60 62 60 64 65 0", "singer3 (male)"],
+    ["jp", "い じ ん さ ん に つ れ ら れ て", "0.6 0.3 0.3 0.3 0.3 0.6 0.6 0.3 0.3 0.6 0.23", "60 60 60 56 56 56 55 55 55 53 56", "singer8 (female)"],
+    ["jp", "い じ ん さ ん に つ れ ら れ て", "0.6 0.3 0.3 0.3 0.3 0.6 0.6 0.3 0.3 0.6 0.23", "62 62 62 58 58 58 57 57 57 55 58", "singer8 (female)"],
+    ["jp", "い じ ん さ ん に つ れ ら れ て", "1.2 0.6 0.6 0.6 0.6 1.2 1.2 0.6 0.6 1.2 0.45", "60 60 60 56 56 56 55 55 55 53 56", "singer8 (female)"],
+    ["jp", "い じ ん さ ん に つ れ ら れ て", "0.3 0.15 0.15 0.15 0.15 0.3 0.3 0.15 0.15 0.3 0.11", "60 60 60 56 56 56 55 55 55 53 56", "singer8 (female)"],
     ["jp", "きっ と と べ ば そ ら ま で と ど く AP", "0.39 2.76 0.2 0.2 0.39 0.39 0.2 0.2 0.39 0.2 0.2 0.59 1.08", "64 71 68 69 71 71 69 68 66 68 69 68 0", "singer2 (female)"],
+    ["jp", "じゃ の め で お む か え う れ し い な", "0.43 0.14 0.43 0.14 0.43 0.14 0.43 0.14 0.43 0.14 0.43 0.14 0.65", "60 60 60 62 64 67 69 69 64 64 64 62 60", "singer10 (female)"],
+    ["jp", "お と め わ ら い か ふぁ い や ら い か ん な い す ぶ ろ うぃ ん ぶ ろ うぃ ん い ん ざ うぃ ん", "0.15 0.15 0.15 0.15 0.3 0.15 0.3 0.15 0.15 0.3 0.07 0.07 0.15 0.15 0.15 0.15 0.15 0.15 0.45 0.07 0.07 0.07 0.38 0.07 0.07 0.15 0.15 0.3 0.15 0.15", "67 67 67 67 67 67 69 67 67 69 67 67 64 64 64 64 64 64 62 64 64 62 62 64 64 62 62 59 59 59", "singer9 (male)"],
 ]
 
 app = gr.Interface(

util.py

@@ -2,6 +2,8 @@ import os
 import json
 import warnings
 from typing import List
+from pypinyin import lazy_pinyin
+import re
 
 import pyopenjtalk
 
@@ -21,7 +23,7 @@ def pyopenjtalk_g2p(text) -> List[str]:
             for warning in w:
                 if "No phoneme" in str(warning.message):
                     return False
-                    
+ 
     phones = phones.split(" ")
     return phones
 
@@ -48,6 +50,20 @@ def get_tokenizer(lang):
     elif lang == "jp":
         return pyopenjtalk_g2p
 
+
+def get_pinyin(texts):
+    pinyin_list = lazy_pinyin(texts)
+    text_list = []
+    for text in pinyin_list:
+        if text[0] == "S" or text[0] == "A" or text[0] == '-':
+            sp_strs = re.findall(r'-|AP|SP', text)
+            for phn in sp_strs:
+                text_list.append(phn)
+        else:
+            text_list.append(text)
+    return text_list
+
+
 def load_pitch_dict(file_path = "resource/midi-note.scp"):
     pitch_dict = {}
     with open(file_path, "r", encoding="utf-8") as f: