Update models.py

app.py

@@ -1,16 +1,15 @@
-import os
+import argparse
 import io
+import logging
+import os
+
 import gradio as gr
+import gradio.processing_utils as gr_processing_utils
 import librosa
 import numpy as np
-import utils
-from inference.infer_tool import Svc
-import logging
 import soundfile
-import asyncio
-import argparse
-import edge_tts
-import gradio.processing_utils as gr_processing_utils
+
+from inference.infer_tool import Svc
 
 logging.getLogger('numba').setLevel(logging.WARNING)
 logging.getLogger('markdown_it').setLevel(logging.WARNING)
@@ -62,7 +61,6 @@ if __name__ == '__main__':
     parser.add_argument('--api', action="store_true", default=False)
     parser.add_argument("--share", action="store_true", default=False, help="share gradio app")
     args = parser.parse_args()
-    hubert_model = utils.get_hubert_model().to(args.device)
     models = []
     voices = []
     for f in os.listdir("models"):

hubert/hubert_model.py

@@ -1,222 +0,0 @@
-import copy
-import random
-from typing import Optional, Tuple
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as t_func
-from torch.nn.modules.utils import consume_prefix_in_state_dict_if_present
-
-
-class Hubert(nn.Module):
-    def __init__(self, num_label_embeddings: int = 100, mask: bool = True):
-        super().__init__()
-        self._mask = mask
-        self.feature_extractor = FeatureExtractor()
-        self.feature_projection = FeatureProjection()
-        self.positional_embedding = PositionalConvEmbedding()
-        self.norm = nn.LayerNorm(768)
-        self.dropout = nn.Dropout(0.1)
-        self.encoder = TransformerEncoder(
-            nn.TransformerEncoderLayer(
-                768, 12, 3072, activation="gelu", batch_first=True
-            ),
-            12,
-        )
-        self.proj = nn.Linear(768, 256)
-
-        self.masked_spec_embed = nn.Parameter(torch.FloatTensor(768).uniform_())
-        self.label_embedding = nn.Embedding(num_label_embeddings, 256)
-
-    def mask(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
-        mask = None
-        if self.training and self._mask:
-            mask = _compute_mask((x.size(0), x.size(1)), 0.8, 10, x.device, 2)
-            x[mask] = self.masked_spec_embed.to(x.dtype)
-        return x, mask
-
-    def encode(
-            self, x: torch.Tensor, layer: Optional[int] = None
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        x = self.feature_extractor(x)
-        x = self.feature_projection(x.transpose(1, 2))
-        x, mask = self.mask(x)
-        x = x + self.positional_embedding(x)
-        x = self.dropout(self.norm(x))
-        x = self.encoder(x, output_layer=layer)
-        return x, mask
-
-    def logits(self, x: torch.Tensor) -> torch.Tensor:
-        logits = torch.cosine_similarity(
-            x.unsqueeze(2),
-            self.label_embedding.weight.unsqueeze(0).unsqueeze(0),
-            dim=-1,
-        )
-        return logits / 0.1
-
-    def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
-        x, mask = self.encode(x)
-        x = self.proj(x)
-        logits = self.logits(x)
-        return logits, mask
-
-
-class HubertSoft(Hubert):
-    def __init__(self):
-        super().__init__()
-
-    @torch.inference_mode()
-    def units(self, wav: torch.Tensor) -> torch.Tensor:
-        wav = t_func.pad(wav, ((400 - 320) // 2, (400 - 320) // 2))
-        x, _ = self.encode(wav)
-        return self.proj(x)
-
-
-class FeatureExtractor(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.conv0 = nn.Conv1d(1, 512, 10, 5, bias=False)
-        self.norm0 = nn.GroupNorm(512, 512)
-        self.conv1 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv2 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv3 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv4 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv5 = nn.Conv1d(512, 512, 2, 2, bias=False)
-        self.conv6 = nn.Conv1d(512, 512, 2, 2, bias=False)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = t_func.gelu(self.norm0(self.conv0(x)))
-        x = t_func.gelu(self.conv1(x))
-        x = t_func.gelu(self.conv2(x))
-        x = t_func.gelu(self.conv3(x))
-        x = t_func.gelu(self.conv4(x))
-        x = t_func.gelu(self.conv5(x))
-        x = t_func.gelu(self.conv6(x))
-        return x
-
-
-class FeatureProjection(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.norm = nn.LayerNorm(512)
-        self.projection = nn.Linear(512, 768)
-        self.dropout = nn.Dropout(0.1)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = self.norm(x)
-        x = self.projection(x)
-        x = self.dropout(x)
-        return x
-
-
-class PositionalConvEmbedding(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.conv = nn.Conv1d(
-            768,
-            768,
-            kernel_size=128,
-            padding=128 // 2,
-            groups=16,
-        )
-        self.conv = nn.utils.weight_norm(self.conv, name="weight", dim=2)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = self.conv(x.transpose(1, 2))
-        x = t_func.gelu(x[:, :, :-1])
-        return x.transpose(1, 2)
-
-
-class TransformerEncoder(nn.Module):
-    def __init__(
-            self, encoder_layer: nn.TransformerEncoderLayer, num_layers: int
-    ) -> None:
-        super(TransformerEncoder, self).__init__()
-        self.layers = nn.ModuleList(
-            [copy.deepcopy(encoder_layer) for _ in range(num_layers)]
-        )
-        self.num_layers = num_layers
-
-    def forward(
-            self,
-            src: torch.Tensor,
-            mask: torch.Tensor = None,
-            src_key_padding_mask: torch.Tensor = None,
-            output_layer: Optional[int] = None,
-    ) -> torch.Tensor:
-        output = src
-        for layer in self.layers[:output_layer]:
-            output = layer(
-                output, src_mask=mask, src_key_padding_mask=src_key_padding_mask
-            )
-        return output
-
-
-def _compute_mask(
-        shape: Tuple[int, int],
-        mask_prob: float,
-        mask_length: int,
-        device: torch.device,
-        min_masks: int = 0,
-) -> torch.Tensor:
-    batch_size, sequence_length = shape
-
-    if mask_length < 1:
-        raise ValueError("`mask_length` has to be bigger than 0.")
-
-    if mask_length > sequence_length:
-        raise ValueError(
-            f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length} and `sequence_length`: {sequence_length}`"
-        )
-
-    # compute number of masked spans in batch
-    num_masked_spans = int(mask_prob * sequence_length / mask_length + random.random())
-    num_masked_spans = max(num_masked_spans, min_masks)
-
-    # make sure num masked indices <= sequence_length
-    if num_masked_spans * mask_length > sequence_length:
-        num_masked_spans = sequence_length // mask_length
-
-    # SpecAugment mask to fill
-    mask = torch.zeros((batch_size, sequence_length), device=device, dtype=torch.bool)
-
-    # uniform distribution to sample from, make sure that offset samples are < sequence_length
-    uniform_dist = torch.ones(
-        (batch_size, sequence_length - (mask_length - 1)), device=device
-    )
-
-    # get random indices to mask
-    mask_indices = torch.multinomial(uniform_dist, num_masked_spans)
-
-    # expand masked indices to masked spans
-    mask_indices = (
-        mask_indices.unsqueeze(dim=-1)
-        .expand((batch_size, num_masked_spans, mask_length))
-        .reshape(batch_size, num_masked_spans * mask_length)
-    )
-    offsets = (
-        torch.arange(mask_length, device=device)[None, None, :]
-        .expand((batch_size, num_masked_spans, mask_length))
-        .reshape(batch_size, num_masked_spans * mask_length)
-    )
-    mask_idxs = mask_indices + offsets
-
-    # scatter indices to mask
-    mask = mask.scatter(1, mask_idxs, True)
-
-    return mask
-
-
-def hubert_soft(
-        path: str,
-) -> HubertSoft:
-    r"""HuBERT-Soft from `"A Comparison of Discrete and Soft Speech Units for Improved Voice Conversion"`.
-    Args:
-        path (str): path of a pretrained model
-    """
-    hubert = HubertSoft()
-    checkpoint = torch.load(path)
-    consume_prefix_in_state_dict_if_present(checkpoint, "module.")
-    hubert.load_state_dict(checkpoint)
-    hubert.eval()
-    return hubert

hubert/hubert_model_onnx.py

@@ -1,217 +0,0 @@
-import copy
-import random
-from typing import Optional, Tuple
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as t_func
-from torch.nn.modules.utils import consume_prefix_in_state_dict_if_present
-
-
-class Hubert(nn.Module):
-    def __init__(self, num_label_embeddings: int = 100, mask: bool = True):
-        super().__init__()
-        self._mask = mask
-        self.feature_extractor = FeatureExtractor()
-        self.feature_projection = FeatureProjection()
-        self.positional_embedding = PositionalConvEmbedding()
-        self.norm = nn.LayerNorm(768)
-        self.dropout = nn.Dropout(0.1)
-        self.encoder = TransformerEncoder(
-            nn.TransformerEncoderLayer(
-                768, 12, 3072, activation="gelu", batch_first=True
-            ),
-            12,
-        )
-        self.proj = nn.Linear(768, 256)
-
-        self.masked_spec_embed = nn.Parameter(torch.FloatTensor(768).uniform_())
-        self.label_embedding = nn.Embedding(num_label_embeddings, 256)
-
-    def mask(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
-        mask = None
-        if self.training and self._mask:
-            mask = _compute_mask((x.size(0), x.size(1)), 0.8, 10, x.device, 2)
-            x[mask] = self.masked_spec_embed.to(x.dtype)
-        return x, mask
-
-    def encode(
-            self, x: torch.Tensor, layer: Optional[int] = None
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        x = self.feature_extractor(x)
-        x = self.feature_projection(x.transpose(1, 2))
-        x, mask = self.mask(x)
-        x = x + self.positional_embedding(x)
-        x = self.dropout(self.norm(x))
-        x = self.encoder(x, output_layer=layer)
-        return x, mask
-
-    def logits(self, x: torch.Tensor) -> torch.Tensor:
-        logits = torch.cosine_similarity(
-            x.unsqueeze(2),
-            self.label_embedding.weight.unsqueeze(0).unsqueeze(0),
-            dim=-1,
-        )
-        return logits / 0.1
-
-
-class HubertSoft(Hubert):
-    def __init__(self):
-        super().__init__()
-
-    def units(self, wav: torch.Tensor) -> torch.Tensor:
-        wav = t_func.pad(wav, ((400 - 320) // 2, (400 - 320) // 2))
-        x, _ = self.encode(wav)
-        return self.proj(x)
-
-    def forward(self, x):
-        return self.units(x)
-
-class FeatureExtractor(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.conv0 = nn.Conv1d(1, 512, 10, 5, bias=False)
-        self.norm0 = nn.GroupNorm(512, 512)
-        self.conv1 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv2 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv3 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv4 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv5 = nn.Conv1d(512, 512, 2, 2, bias=False)
-        self.conv6 = nn.Conv1d(512, 512, 2, 2, bias=False)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = t_func.gelu(self.norm0(self.conv0(x)))
-        x = t_func.gelu(self.conv1(x))
-        x = t_func.gelu(self.conv2(x))
-        x = t_func.gelu(self.conv3(x))
-        x = t_func.gelu(self.conv4(x))
-        x = t_func.gelu(self.conv5(x))
-        x = t_func.gelu(self.conv6(x))
-        return x
-
-
-class FeatureProjection(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.norm = nn.LayerNorm(512)
-        self.projection = nn.Linear(512, 768)
-        self.dropout = nn.Dropout(0.1)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = self.norm(x)
-        x = self.projection(x)
-        x = self.dropout(x)
-        return x
-
-
-class PositionalConvEmbedding(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.conv = nn.Conv1d(
-            768,
-            768,
-            kernel_size=128,
-            padding=128 // 2,
-            groups=16,
-        )
-        self.conv = nn.utils.weight_norm(self.conv, name="weight", dim=2)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = self.conv(x.transpose(1, 2))
-        x = t_func.gelu(x[:, :, :-1])
-        return x.transpose(1, 2)
-
-
-class TransformerEncoder(nn.Module):
-    def __init__(
-            self, encoder_layer: nn.TransformerEncoderLayer, num_layers: int
-    ) -> None:
-        super(TransformerEncoder, self).__init__()
-        self.layers = nn.ModuleList(
-            [copy.deepcopy(encoder_layer) for _ in range(num_layers)]
-        )
-        self.num_layers = num_layers
-
-    def forward(
-            self,
-            src: torch.Tensor,
-            mask: torch.Tensor = None,
-            src_key_padding_mask: torch.Tensor = None,
-            output_layer: Optional[int] = None,
-    ) -> torch.Tensor:
-        output = src
-        for layer in self.layers[:output_layer]:
-            output = layer(
-                output, src_mask=mask, src_key_padding_mask=src_key_padding_mask
-            )
-        return output
-
-
-def _compute_mask(
-        shape: Tuple[int, int],
-        mask_prob: float,
-        mask_length: int,
-        device: torch.device,
-        min_masks: int = 0,
-) -> torch.Tensor:
-    batch_size, sequence_length = shape
-
-    if mask_length < 1:
-        raise ValueError("`mask_length` has to be bigger than 0.")
-
-    if mask_length > sequence_length:
-        raise ValueError(
-            f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length} and `sequence_length`: {sequence_length}`"
-        )
-
-    # compute number of masked spans in batch
-    num_masked_spans = int(mask_prob * sequence_length / mask_length + random.random())
-    num_masked_spans = max(num_masked_spans, min_masks)
-
-    # make sure num masked indices <= sequence_length
-    if num_masked_spans * mask_length > sequence_length:
-        num_masked_spans = sequence_length // mask_length
-
-    # SpecAugment mask to fill
-    mask = torch.zeros((batch_size, sequence_length), device=device, dtype=torch.bool)
-
-    # uniform distribution to sample from, make sure that offset samples are < sequence_length
-    uniform_dist = torch.ones(
-        (batch_size, sequence_length - (mask_length - 1)), device=device
-    )
-
-    # get random indices to mask
-    mask_indices = torch.multinomial(uniform_dist, num_masked_spans)
-
-    # expand masked indices to masked spans
-    mask_indices = (
-        mask_indices.unsqueeze(dim=-1)
-        .expand((batch_size, num_masked_spans, mask_length))
-        .reshape(batch_size, num_masked_spans * mask_length)
-    )
-    offsets = (
-        torch.arange(mask_length, device=device)[None, None, :]
-        .expand((batch_size, num_masked_spans, mask_length))
-        .reshape(batch_size, num_masked_spans * mask_length)
-    )
-    mask_idxs = mask_indices + offsets
-
-    # scatter indices to mask
-    mask = mask.scatter(1, mask_idxs, True)
-
-    return mask
-
-
-def hubert_soft(
-        path: str,
-) -> HubertSoft:
-    r"""HuBERT-Soft from `"A Comparison of Discrete and Soft Speech Units for Improved Voice Conversion"`.
-    Args:
-        path (str): path of a pretrained model
-    """
-    hubert = HubertSoft()
-    checkpoint = torch.load(path)
-    consume_prefix_in_state_dict_if_present(checkpoint, "module.")
-    hubert.load_state_dict(checkpoint)
-    hubert.eval()
-    return hubert

inference/infer_tool.py

@@ -82,16 +82,19 @@ def get_end_file(dir_path, end):
 def get_md5(content):
     return hashlib.new("md5", content).hexdigest()
 
+
 def fill_a_to_b(a, b):
     if len(a) < len(b):
         for _ in range(0, len(b) - len(a)):
             a.append(a[0])
 
+
 def mkdir(paths: list):
     for path in paths:
         if not os.path.exists(path):
             os.mkdir(path)
 
+
 def pad_array(arr, target_length):
     current_length = arr.shape[0]
     if current_length >= target_length:
@@ -102,15 +105,17 @@ def pad_array(arr, target_length):
         pad_right = pad_width - pad_left
         padded_arr = np.pad(arr, (pad_left, pad_right), 'constant', constant_values=(0, 0))
         return padded_arr
-    
+
+
 def split_list_by_n(list_collection, n, pre=0):
     for i in range(0, len(list_collection), n):
-        yield list_collection[i-pre if i-pre>=0 else i: i + n]
+        yield list_collection[i - pre if i - pre >= 0 else i: i + n]
 
 
 class F0FilterException(Exception):
     pass
 
+
 class Svc(object):
     def __init__(self, net_g_path, config_path,
                  device=None,
@@ -140,14 +145,14 @@ class Svc(object):
 
         if os.path.exists(cluster_model_path):
             if self.feature_retrieval:
-                with open(cluster_model_path,"rb") as f:
+                with open(cluster_model_path, "rb") as f:
                     self.cluster_model = pickle.load(f)
                 self.big_npy = None
                 self.now_spk_id = -1
             else:
                 self.cluster_model = cluster.get_cluster_model(cluster_model_path)
         else:
-            self.feature_retrieval=False
+            self.feature_retrieval = False
 
     def load_model(self, spk_mix_enable=False):
         # get model configuration
@@ -163,10 +168,12 @@ class Svc(object):
         if spk_mix_enable:
             self.net_g_ms.EnableCharacterMix(len(self.spk2id), self.dev)
 
-    def get_unit_f0(self, wav, tran, cluster_infer_ratio, speaker, f0_filter ,f0_predictor,cr_threshold=0.05):
+    def get_unit_f0(self, wav, tran, cluster_infer_ratio, speaker, f0_filter, f0_predictor, cr_threshold=0.05):
+
+        f0_predictor_object = utils.get_f0_predictor(f0_predictor, hop_length=self.hop_size,
+                                                     sampling_rate=self.target_sample, device=self.dev,
+                                                     threshold=cr_threshold)
 
-        f0_predictor_object = utils.get_f0_predictor(f0_predictor,hop_length=self.hop_size,sampling_rate=self.target_sample,device=self.dev,threshold=cr_threshold)
-        
         f0, uv = f0_predictor_object.compute_f0_uv(wav)
         if f0_filter and sum(f0) == 0:
             raise F0FilterException("No voice detected")
@@ -179,10 +186,11 @@ class Svc(object):
 
         wav16k = librosa.resample(wav, orig_sr=self.target_sample, target_sr=16000)
         wav16k = torch.from_numpy(wav16k).to(self.dev)
-        c = utils.get_hubert_content(self.hubert_model, wav_16k_tensor=wav16k)
+        c = self.hubert_model.encoder(wav16k)
+        # c = utils.get_hubert_content(self.hubert_model, wav_16k_tensor=wav16k)
         c = utils.repeat_expand_2d(c.squeeze(0), f0.shape[1])
 
-        if cluster_infer_ratio !=0:
+        if cluster_infer_ratio != 0:
             if self.feature_retrieval:
                 speaker_id = self.spk2id.get(speaker)
                 if speaker_id is None:
@@ -191,17 +199,17 @@ class Svc(object):
                     if len(self.spk2id.__dict__) >= speaker:
                         speaker_id = speaker
                 feature_index = self.cluster_model[speaker_id]
-                feat_np = c.transpose(0,1).cpu().numpy()
+                feat_np = c.transpose(0, 1).cpu().numpy()
                 if self.big_npy is None or self.now_spk_id != speaker_id:
-                   self.big_npy = feature_index.reconstruct_n(0, feature_index.ntotal)
-                   self.now_spk_id = speaker_id
+                    self.big_npy = feature_index.reconstruct_n(0, feature_index.ntotal)
+                    self.now_spk_id = speaker_id
                 print("starting feature retrieval...")
                 score, ix = feature_index.search(feat_np, k=8)
                 weight = np.square(1 / score)
                 weight /= weight.sum(axis=1, keepdims=True)
                 npy = np.sum(self.big_npy[ix] * np.expand_dims(weight, axis=2), axis=1)
                 c = cluster_infer_ratio * npy + (1 - cluster_infer_ratio) * feat_np
-                c = torch.FloatTensor(c).to(self.dev).transpose(0,1)
+                c = torch.FloatTensor(c).to(self.dev).transpose(0, 1)
                 print("end feature retrieval...")
             else:
                 cluster_c = cluster.get_cluster_center_result(self.cluster_model, c.cpu().numpy().T, speaker).T
@@ -217,41 +225,35 @@ class Svc(object):
               noice_scale=0.4,
               f0_filter=False,
               f0_predictor='pm',
-              enhancer_adaptive_key = 0,
-              cr_threshold = 0.05,
-              k_step = 100,
-              frame = 0,
-              spk_mix = False,
-              second_encoding = False,
-              loudness_envelope_adjustment = 1
+              enhancer_adaptive_key=0,
+              cr_threshold=0.05,
+              k_step=100,
+              frame=0,
+              spk_mix=False,
+              second_encoding=False,
+              loudness_envelope_adjustment=1
               ):
         wav, sr = librosa.load(raw_path, sr=self.target_sample)
-        if spk_mix:
-            c, f0, uv = self.get_unit_f0(wav, tran, 0, None, f0_filter,f0_predictor,cr_threshold=cr_threshold)
-            n_frames = f0.size(1)
-            sid = speaker[:, frame:frame+n_frames].transpose(0,1)
-        else:
-            speaker_id = self.spk2id.get(speaker)
-            if not speaker_id and type(speaker) is int:
-                if len(self.spk2id.__dict__) >= speaker:
-                    speaker_id = speaker
-            if speaker_id is None:
-                raise RuntimeError("The name you entered is not in the speaker list!")
-            sid = torch.LongTensor([int(speaker_id)]).to(self.dev).unsqueeze(0)
-            c, f0, uv = self.get_unit_f0(wav, tran, cluster_infer_ratio, speaker, f0_filter,f0_predictor,cr_threshold=cr_threshold)
-            n_frames = f0.size(1)
+        speaker_id = self.spk2id.get(speaker)
+        if not speaker_id and type(speaker) is int:
+            if len(self.spk2id.__dict__) >= speaker:
+                speaker_id = speaker
+        if speaker_id is None:
+            raise RuntimeError("The name you entered is not in the speaker list!")
+        sid = torch.LongTensor([int(speaker_id)]).to(self.dev).unsqueeze(0)
+        c, f0, uv = self.get_unit_f0(wav, tran, cluster_infer_ratio, speaker, f0_filter, f0_predictor,
+                                     cr_threshold=cr_threshold)
+        n_frames = f0.size(1)
         if "half" in self.net_g_path and torch.cuda.is_available():
             c = c.half()
         with torch.no_grad():
             start = time.time()
             vol = None
-            vol = self.volume_extractor.extract(torch.FloatTensor(wav).to(self.dev)[None,:])[None,:].to(self.dev) if self.vol_embedding else None
-            audio,f0 = self.net_g_ms.infer(c, f0=f0, g=sid, uv=uv, predict_f0=auto_predict_f0, noice_scale=noice_scale,vol=vol)
-            audio = audio[0,0].data.float()
-            audio_mel = self.vocoder.extract(audio[None,:],self.target_sample) if self.shallow_diffusion else None
-
-            if loudness_envelope_adjustment != 1:
-                audio = utils.change_rms(wav,self.target_sample,audio,self.target_sample,loudness_envelope_adjustment)
+            vol = self.volume_extractor.extract(torch.FloatTensor(wav).to(self.dev)[None, :])[None, :].to(
+                self.dev) if self.vol_embedding else None
+            audio, f0 = self.net_g_ms.infer(c, f0=f0, g=sid, uv=uv, predict_f0=auto_predict_f0, noice_scale=noice_scale,
+                                            vol=vol)
+            audio = audio[0, 0].data.float()
             use_time = time.time() - start
             print("vits use time:{}".format(use_time))
         return audio, audio.shape[-1], n_frames
@@ -264,7 +266,7 @@ class Svc(object):
         # unload model
         self.net_g_ms = self.net_g_ms.to("cpu")
         del self.net_g_ms
-        if hasattr(self,"enhancer"): 
+        if hasattr(self, "enhancer"):
             self.enhancer.enhancer = self.enhancer.enhancer.to("cpu")
             del self.enhancer.enhancer
             del self.enhancer
@@ -281,14 +283,14 @@ class Svc(object):
                         pad_seconds=0.5,
                         clip_seconds=0,
                         lg_num=0,
-                        lgr_num =0.75,
+                        lgr_num=0.75,
                         f0_predictor='pm',
-                        enhancer_adaptive_key = 0,
-                        cr_threshold = 0.05,
-                        k_step = 100,
-                        use_spk_mix = False,
-                        second_encoding = False,
-                        loudness_envelope_adjustment = 1
+                        enhancer_adaptive_key=0,
+                        cr_threshold=0.05,
+                        k_step=100,
+                        use_spk_mix=False,
+                        second_encoding=False,
+                        loudness_envelope_adjustment=1
                         ):
         if use_spk_mix:
             if len(self.spk2id) == 1:
@@ -297,12 +299,12 @@ class Svc(object):
         wav_path = Path(raw_audio_path).with_suffix('.wav')
         chunks = slicer.cut(wav_path, db_thresh=slice_db)
         audio_data, audio_sr = slicer.chunks2audio(wav_path, chunks)
-        per_size = int(clip_seconds*audio_sr)
-        lg_size = int(lg_num*audio_sr)
-        lg_size_r = int(lg_size*lgr_num)
-        lg_size_c_l = (lg_size-lg_size_r)//2
-        lg_size_c_r = lg_size-lg_size_r-lg_size_c_l
-        lg = np.linspace(0,1,lg_size_r) if lg_size!=0 else 0
+        per_size = int(clip_seconds * audio_sr)
+        lg_size = int(lg_num * audio_sr)
+        lg_size_r = int(lg_size * lgr_num)
+        lg_size_c_l = (lg_size - lg_size_r) // 2
+        lg_size_c_r = lg_size - lg_size_r - lg_size_c_l
+        lg = np.linspace(0, 1, lg_size_r) if lg_size != 0 else 0
 
         if use_spk_mix:
             assert len(self.spk2id) == len(spk)
@@ -313,10 +315,10 @@ class Svc(object):
                     audio_length += aud_length // self.hop_size
                     continue
                 if per_size != 0:
-                    datas = split_list_by_n(data, per_size,lg_size)
+                    datas = split_list_by_n(data, per_size, lg_size)
                 else:
                     datas = [data]
-                for k,dat in enumerate(datas):
+                for k, dat in enumerate(datas):
                     pad_len = int(audio_sr * pad_seconds)
                     per_length = int(np.ceil(len(dat) / audio_sr * self.target_sample))
                     a_length = per_length + 2 * pad_len
@@ -326,14 +328,14 @@ class Svc(object):
             for i in range(len(spk)):
                 last_end = None
                 for mix in spk[i]:
-                    if mix[3]<0. or mix[2]<0.:
+                    if mix[3] < 0. or mix[2] < 0.:
                         raise RuntimeError("mix value must higer Than zero!")
                     begin = int(audio_length * mix[0])
                     end = int(audio_length * mix[1])
                     length = end - begin
-                    if length<=0:                        
+                    if length <= 0:
                         raise RuntimeError("begin Must lower Than end!")
-                    step = (mix[3] - mix[2])/length
+                    step = (mix[3] - mix[2]) / length
                     if last_end is not None:
                         if last_end != begin:
                             raise RuntimeError("[i]EndTime Must Equal [i+1]BeginTime!")
@@ -341,20 +343,20 @@ class Svc(object):
                     if step == 0.:
                         spk_mix_data = torch.zeros(length).to(self.dev) + mix[2]
                     else:
-                        spk_mix_data = torch.arange(mix[2],mix[3],step).to(self.dev)
-                    if(len(spk_mix_data)<length):
+                        spk_mix_data = torch.arange(mix[2], mix[3], step).to(self.dev)
+                    if (len(spk_mix_data) < length):
                         num_pad = length - len(spk_mix_data)
                         spk_mix_data = torch.nn.functional.pad(spk_mix_data, [0, num_pad], mode="reflect").to(self.dev)
                     spk_mix_tensor[i][begin:end] = spk_mix_data[:length]
 
-            spk_mix_ten = torch.sum(spk_mix_tensor,dim=0).unsqueeze(0).to(self.dev)
+            spk_mix_ten = torch.sum(spk_mix_tensor, dim=0).unsqueeze(0).to(self.dev)
             # spk_mix_tensor[0][spk_mix_ten<0.001] = 1.0
             for i, x in enumerate(spk_mix_ten[0]):
                 if x == 0.0:
                     spk_mix_ten[0][i] = 1.0
-                    spk_mix_tensor[:,i] = 1.0 / len(spk)
+                    spk_mix_tensor[:, i] = 1.0 / len(spk)
             spk_mix_tensor = spk_mix_tensor / spk_mix_ten
-            if not ((torch.sum(spk_mix_tensor,dim=0) - 1.)<0.0001).all():
+            if not ((torch.sum(spk_mix_tensor, dim=0) - 1.) < 0.0001).all():
                 raise RuntimeError("sum(spk_mix_tensor) not equal 1")
             spk = spk_mix_tensor
 
@@ -371,12 +373,12 @@ class Svc(object):
                 global_frame += length // self.hop_size
                 continue
             if per_size != 0:
-                datas = split_list_by_n(data, per_size,lg_size)
+                datas = split_list_by_n(data, per_size, lg_size)
             else:
                 datas = [data]
-            for k,dat in enumerate(datas):
-                per_length = int(np.ceil(len(dat) / audio_sr * self.target_sample)) if clip_seconds!=0 else length
-                if clip_seconds!=0: print(f'###=====segment clip start, {round(len(dat) / audio_sr, 3)}s======')
+            for k, dat in enumerate(datas):
+                per_length = int(np.ceil(len(dat) / audio_sr * self.target_sample)) if clip_seconds != 0 else length
+                if clip_seconds != 0: print(f'###=====segment clip start, {round(len(dat) / audio_sr, 3)}s======')
                 # padd
                 pad_len = int(audio_sr * pad_seconds)
                 dat = np.concatenate([np.zeros([pad_len]), dat, np.zeros([pad_len])])
@@ -384,33 +386,34 @@ class Svc(object):
                 soundfile.write(raw_path, dat, audio_sr, format="wav")
                 raw_path.seek(0)
                 out_audio, out_sr, out_frame = self.infer(spk, tran, raw_path,
-                                                    cluster_infer_ratio=cluster_infer_ratio,
-                                                    auto_predict_f0=auto_predict_f0,
-                                                    noice_scale=noice_scale,
-                                                    f0_predictor = f0_predictor,
-                                                    enhancer_adaptive_key = enhancer_adaptive_key,
-                                                    cr_threshold = cr_threshold,
-                                                    k_step = k_step,
-                                                    frame = global_frame,
-                                                    spk_mix = use_spk_mix,
-                                                    second_encoding = second_encoding,
-                                                    loudness_envelope_adjustment = loudness_envelope_adjustment
-                                                    )
+                                                          cluster_infer_ratio=cluster_infer_ratio,
+                                                          auto_predict_f0=auto_predict_f0,
+                                                          noice_scale=noice_scale,
+                                                          f0_predictor=f0_predictor,
+                                                          enhancer_adaptive_key=enhancer_adaptive_key,
+                                                          cr_threshold=cr_threshold,
+                                                          k_step=k_step,
+                                                          frame=global_frame,
+                                                          spk_mix=use_spk_mix,
+                                                          second_encoding=second_encoding,
+                                                          loudness_envelope_adjustment=loudness_envelope_adjustment
+                                                          )
                 global_frame += out_frame
                 _audio = out_audio.cpu().numpy()
                 pad_len = int(self.target_sample * pad_seconds)
                 _audio = _audio[pad_len:-pad_len]
                 _audio = pad_array(_audio, per_length)
-                if lg_size!=0 and k!=0:
-                    lg1 = audio[-(lg_size_r+lg_size_c_r):-lg_size_c_r] if lgr_num != 1 else audio[-lg_size:]
-                    lg2 = _audio[lg_size_c_l:lg_size_c_l+lg_size_r]  if lgr_num != 1 else _audio[0:lg_size]
-                    lg_pre = lg1*(1-lg)+lg2*lg
-                    audio = audio[0:-(lg_size_r+lg_size_c_r)] if lgr_num != 1 else audio[0:-lg_size]
+                if lg_size != 0 and k != 0:
+                    lg1 = audio[-(lg_size_r + lg_size_c_r):-lg_size_c_r] if lgr_num != 1 else audio[-lg_size:]
+                    lg2 = _audio[lg_size_c_l:lg_size_c_l + lg_size_r] if lgr_num != 1 else _audio[0:lg_size]
+                    lg_pre = lg1 * (1 - lg) + lg2 * lg
+                    audio = audio[0:-(lg_size_r + lg_size_c_r)] if lgr_num != 1 else audio[0:-lg_size]
                     audio.extend(lg_pre)
-                    _audio = _audio[lg_size_c_l+lg_size_r:] if lgr_num != 1 else _audio[lg_size:]
+                    _audio = _audio[lg_size_c_l + lg_size_r:] if lgr_num != 1 else _audio[lg_size:]
                 audio.extend(list(_audio))
         return np.array(audio)
 
+
 class RealTimeVC:
     def __init__(self):
         self.last_chunk = None
@@ -438,7 +441,7 @@ class RealTimeVC:
                                         auto_predict_f0=auto_predict_f0,
                                         noice_scale=noice_scale,
                                         f0_filter=f0_filter)
-            
+
             audio = audio.cpu().numpy()
             self.last_chunk = audio[-self.pre_len:]
             self.last_o = audio
@@ -459,4 +462,3 @@ class RealTimeVC:
             self.last_chunk = audio[-self.pre_len:]
             self.last_o = audio
             return ret[self.chunk_len:2 * self.chunk_len]
-            

inference/inference/chunks_temp.json

@@ -0,0 +1 @@
+{"info": "temp_dict"}

inference/inference/infer_tool.py

@@ -0,0 +1,533 @@
+import hashlib
+import io
+import json
+import logging
+import os
+import time
+from pathlib import Path
+from inference import slicer
+import gc
+
+import librosa
+import numpy as np
+# import onnxruntime
+import soundfile
+import torch
+import torchaudio
+
+import cluster
+import utils
+from models import SynthesizerTrn
+import pickle
+
+from diffusion.unit2mel import load_model_vocoder
+import yaml
+
+logging.getLogger('matplotlib').setLevel(logging.WARNING)
+
+
+def read_temp(file_name):
+    if not os.path.exists(file_name):
+        with open(file_name, "w") as f:
+            f.write(json.dumps({"info": "temp_dict"}))
+        return {}
+    else:
+        try:
+            with open(file_name, "r") as f:
+                data = f.read()
+            data_dict = json.loads(data)
+            if os.path.getsize(file_name) > 50 * 1024 * 1024:
+                f_name = file_name.replace("\\", "/").split("/")[-1]
+                print(f"clean {f_name}")
+                for wav_hash in list(data_dict.keys()):
+                    if int(time.time()) - int(data_dict[wav_hash]["time"]) > 14 * 24 * 3600:
+                        del data_dict[wav_hash]
+        except Exception as e:
+            print(e)
+            print(f"{file_name} error,auto rebuild file")
+            data_dict = {"info": "temp_dict"}
+        return data_dict
+
+
+def write_temp(file_name, data):
+    with open(file_name, "w") as f:
+        f.write(json.dumps(data))
+
+
+def timeit(func):
+    def run(*args, **kwargs):
+        t = time.time()
+        res = func(*args, **kwargs)
+        print('executing \'%s\' costed %.3fs' % (func.__name__, time.time() - t))
+        return res
+
+    return run
+
+
+def format_wav(audio_path):
+    if Path(audio_path).suffix == '.wav':
+        return
+    raw_audio, raw_sample_rate = librosa.load(audio_path, mono=True, sr=None)
+    soundfile.write(Path(audio_path).with_suffix(".wav"), raw_audio, raw_sample_rate)
+
+
+def get_end_file(dir_path, end):
+    file_lists = []
+    for root, dirs, files in os.walk(dir_path):
+        files = [f for f in files if f[0] != '.']
+        dirs[:] = [d for d in dirs if d[0] != '.']
+        for f_file in files:
+            if f_file.endswith(end):
+                file_lists.append(os.path.join(root, f_file).replace("\\", "/"))
+    return file_lists
+
+
+def get_md5(content):
+    return hashlib.new("md5", content).hexdigest()
+
+def fill_a_to_b(a, b):
+    if len(a) < len(b):
+        for _ in range(0, len(b) - len(a)):
+            a.append(a[0])
+
+def mkdir(paths: list):
+    for path in paths:
+        if not os.path.exists(path):
+            os.mkdir(path)
+
+def pad_array(arr, target_length):
+    current_length = arr.shape[0]
+    if current_length >= target_length:
+        return arr
+    else:
+        pad_width = target_length - current_length
+        pad_left = pad_width // 2
+        pad_right = pad_width - pad_left
+        padded_arr = np.pad(arr, (pad_left, pad_right), 'constant', constant_values=(0, 0))
+        return padded_arr
+    
+def split_list_by_n(list_collection, n, pre=0):
+    for i in range(0, len(list_collection), n):
+        yield list_collection[i-pre if i-pre>=0 else i: i + n]
+
+
+class F0FilterException(Exception):
+    pass
+
+class Svc(object):
+    def __init__(self, net_g_path, config_path,
+                 device=None,
+                 cluster_model_path="logs/44k/kmeans_10000.pt",
+                 nsf_hifigan_enhance = False,
+                 diffusion_model_path="logs/44k/diffusion/model_0.pt",
+                 diffusion_config_path="configs/diffusion.yaml",
+                 shallow_diffusion = False,
+                 only_diffusion = False,
+                 spk_mix_enable = False,
+                 feature_retrieval = False
+                 ):
+        self.net_g_path = net_g_path
+        self.only_diffusion = only_diffusion
+        self.shallow_diffusion = shallow_diffusion
+        self.feature_retrieval = feature_retrieval
+        if device is None:
+            self.dev = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        else:
+            self.dev = torch.device(device)
+        self.net_g_ms = None
+        if not self.only_diffusion:
+            self.hps_ms = utils.get_hparams_from_file(config_path)
+            self.target_sample = self.hps_ms.data.sampling_rate
+            self.hop_size = self.hps_ms.data.hop_length
+            self.spk2id = self.hps_ms.spk
+            try:
+                self.vol_embedding = self.hps_ms.model.vol_embedding
+            except Exception as e:
+                self.vol_embedding = False
+            try:
+                self.speech_encoder = self.hps_ms.model.speech_encoder
+            except Exception as e:
+                self.speech_encoder = 'vec768l12'
+
+        self.nsf_hifigan_enhance = nsf_hifigan_enhance
+        if self.shallow_diffusion or self.only_diffusion:
+            if os.path.exists(diffusion_model_path) and os.path.exists(diffusion_model_path):
+                self.diffusion_model,self.vocoder,self.diffusion_args = load_model_vocoder(diffusion_model_path,self.dev,config_path=diffusion_config_path)
+                if self.only_diffusion:
+                    self.target_sample = self.diffusion_args.data.sampling_rate
+                    self.hop_size = self.diffusion_args.data.block_size
+                    self.spk2id = self.diffusion_args.spk
+                    self.speech_encoder = self.diffusion_args.data.encoder
+                if spk_mix_enable:
+                    self.diffusion_model.init_spkmix(len(self.spk2id))
+            else:
+                print("No diffusion model or config found. Shallow diffusion mode will False")
+                self.shallow_diffusion = self.only_diffusion = False
+                
+        # load hubert and model
+        if not self.only_diffusion:
+            self.load_model(spk_mix_enable)
+            self.hubert_model = utils.get_speech_encoder(self.speech_encoder,device=self.dev)
+            self.volume_extractor = utils.Volume_Extractor(self.hop_size)
+        else:
+            self.hubert_model = utils.get_speech_encoder(self.diffusion_args.data.encoder,device=self.dev)
+            self.volume_extractor = utils.Volume_Extractor(self.diffusion_args.data.block_size)
+            
+        if os.path.exists(cluster_model_path):
+            if self.feature_retrieval:
+                with open(cluster_model_path,"rb") as f:
+                    self.cluster_model = pickle.load(f)
+                self.big_npy = None
+                self.now_spk_id = -1
+            else:
+                self.cluster_model = cluster.get_cluster_model(cluster_model_path)
+        else:
+            self.feature_retrieval=False
+
+        if self.shallow_diffusion : self.nsf_hifigan_enhance = False
+        if self.nsf_hifigan_enhance:
+            from modules.enhancer import Enhancer
+            self.enhancer = Enhancer('nsf-hifigan', 'pretrain/nsf_hifigan/model',device=self.dev)
+            
+    def load_model(self, spk_mix_enable=False):
+        # get model configuration
+        self.net_g_ms = SynthesizerTrn(
+            self.hps_ms.data.filter_length // 2 + 1,
+            self.hps_ms.train.segment_size // self.hps_ms.data.hop_length,
+            **self.hps_ms.model)
+        _ = utils.load_checkpoint(self.net_g_path, self.net_g_ms, None)
+        if "half" in self.net_g_path and torch.cuda.is_available():
+            _ = self.net_g_ms.half().eval().to(self.dev)
+        else:
+            _ = self.net_g_ms.eval().to(self.dev)
+        if spk_mix_enable:
+            self.net_g_ms.EnableCharacterMix(len(self.spk2id), self.dev)
+
+    def get_unit_f0(self, wav, tran, cluster_infer_ratio, speaker, f0_filter ,f0_predictor,cr_threshold=0.05):
+
+        f0_predictor_object = utils.get_f0_predictor(f0_predictor,hop_length=self.hop_size,sampling_rate=self.target_sample,device=self.dev,threshold=cr_threshold)
+        
+        f0, uv = f0_predictor_object.compute_f0_uv(wav)
+        if f0_filter and sum(f0) == 0:
+            raise F0FilterException("No voice detected")
+        f0 = torch.FloatTensor(f0).to(self.dev)
+        uv = torch.FloatTensor(uv).to(self.dev)
+
+        f0 = f0 * 2 ** (tran / 12)
+        f0 = f0.unsqueeze(0)
+        uv = uv.unsqueeze(0)
+
+        wav16k = librosa.resample(wav, orig_sr=self.target_sample, target_sr=16000)
+        wav16k = torch.from_numpy(wav16k).to(self.dev)
+        c = self.hubert_model.encoder(wav16k)
+        c = utils.repeat_expand_2d(c.squeeze(0), f0.shape[1])
+
+        if cluster_infer_ratio !=0:
+            if self.feature_retrieval:
+                speaker_id = self.spk2id.get(speaker)
+                if speaker_id is None:
+                    raise RuntimeError("The name you entered is not in the speaker list!")
+                if not speaker_id and type(speaker) is int:
+                    if len(self.spk2id.__dict__) >= speaker:
+                        speaker_id = speaker
+                feature_index = self.cluster_model[speaker_id]
+                feat_np = c.transpose(0,1).cpu().numpy()
+                if self.big_npy is None or self.now_spk_id != speaker_id:
+                   self.big_npy = feature_index.reconstruct_n(0, feature_index.ntotal)
+                   self.now_spk_id = speaker_id
+                print("starting feature retrieval...")
+                score, ix = feature_index.search(feat_np, k=8)
+                weight = np.square(1 / score)
+                weight /= weight.sum(axis=1, keepdims=True)
+                npy = np.sum(self.big_npy[ix] * np.expand_dims(weight, axis=2), axis=1)
+                c = cluster_infer_ratio * npy + (1 - cluster_infer_ratio) * feat_np
+                c = torch.FloatTensor(c).to(self.dev).transpose(0,1)
+                print("end feature retrieval...")
+            else:
+                cluster_c = cluster.get_cluster_center_result(self.cluster_model, c.cpu().numpy().T, speaker).T
+                cluster_c = torch.FloatTensor(cluster_c).to(self.dev)
+                c = cluster_infer_ratio * cluster_c + (1 - cluster_infer_ratio) * c
+
+        c = c.unsqueeze(0)
+        return c, f0, uv
+
+    def infer(self, speaker, tran, raw_path,
+              cluster_infer_ratio=0,
+              auto_predict_f0=False,
+              noice_scale=0.4,
+              f0_filter=False,
+              f0_predictor='pm',
+              enhancer_adaptive_key = 0,
+              cr_threshold = 0.05,
+              k_step = 100,
+              frame = 0,
+              spk_mix = False,
+              second_encoding = False,
+              loudness_envelope_adjustment = 1
+              ):
+        wav, sr = librosa.load(raw_path, sr=self.target_sample)
+        if spk_mix:
+            c, f0, uv = self.get_unit_f0(wav, tran, 0, None, f0_filter,f0_predictor,cr_threshold=cr_threshold)
+            n_frames = f0.size(1)
+            sid = speaker[:, frame:frame+n_frames].transpose(0,1)
+        else:
+            speaker_id = self.spk2id.get(speaker)
+            if not speaker_id and type(speaker) is int:
+                if len(self.spk2id.__dict__) >= speaker:
+                    speaker_id = speaker
+            if speaker_id is None:
+                raise RuntimeError("The name you entered is not in the speaker list!")
+            sid = torch.LongTensor([int(speaker_id)]).to(self.dev).unsqueeze(0)
+            c, f0, uv = self.get_unit_f0(wav, tran, cluster_infer_ratio, speaker, f0_filter,f0_predictor,cr_threshold=cr_threshold)
+            n_frames = f0.size(1)
+        if "half" in self.net_g_path and torch.cuda.is_available():
+            c = c.half()
+        with torch.no_grad():
+            start = time.time()
+            vol = None
+            if not self.only_diffusion:
+                vol = self.volume_extractor.extract(torch.FloatTensor(wav).to(self.dev)[None,:])[None,:].to(self.dev) if self.vol_embedding else None
+                audio,f0 = self.net_g_ms.infer(c, f0=f0, g=sid, uv=uv, predict_f0=auto_predict_f0, noice_scale=noice_scale,vol=vol)
+                audio = audio[0,0].data.float()
+                audio_mel = self.vocoder.extract(audio[None,:],self.target_sample) if self.shallow_diffusion else None
+            else:
+                audio = torch.FloatTensor(wav).to(self.dev)
+                audio_mel = None
+            if self.only_diffusion or self.shallow_diffusion:
+                vol = self.volume_extractor.extract(audio[None,:])[None,:,None].to(self.dev) if vol==None else vol[:,:,None]
+                if self.shallow_diffusion and second_encoding:
+                    audio16k = librosa.resample(audio.detach().cpu().numpy(), orig_sr=self.target_sample, target_sr=16000)
+                    audio16k = torch.from_numpy(audio16k).to(self.dev)
+                    c = self.hubert_model.encoder(audio16k)
+                    c = utils.repeat_expand_2d(c.squeeze(0), f0.shape[1])
+                f0 = f0[:,:,None]
+                c = c.transpose(-1,-2)
+                audio_mel = self.diffusion_model(
+                c, 
+                f0, 
+                vol, 
+                spk_id = sid, 
+                spk_mix_dict = None,
+                gt_spec=audio_mel,
+                infer=True, 
+                infer_speedup=self.diffusion_args.infer.speedup, 
+                method=self.diffusion_args.infer.method,
+                k_step=k_step)
+                audio = self.vocoder.infer(audio_mel, f0).squeeze()
+            if self.nsf_hifigan_enhance:
+                audio, _ = self.enhancer.enhance(
+                                    audio[None,:], 
+                                    self.target_sample, 
+                                    f0[:,:,None], 
+                                    self.hps_ms.data.hop_length, 
+                                    adaptive_key = enhancer_adaptive_key)
+            if loudness_envelope_adjustment != 1:
+                audio = utils.change_rms(wav,self.target_sample,audio,self.target_sample,loudness_envelope_adjustment)
+            use_time = time.time() - start
+            print("vits use time:{}".format(use_time))
+        return audio, audio.shape[-1], n_frames
+
+    def clear_empty(self):
+        # clean up vram
+        torch.cuda.empty_cache()
+
+    def unload_model(self):
+        # unload model
+        self.net_g_ms = self.net_g_ms.to("cpu")
+        del self.net_g_ms
+        if hasattr(self,"enhancer"): 
+            self.enhancer.enhancer = self.enhancer.enhancer.to("cpu")
+            del self.enhancer.enhancer
+            del self.enhancer
+        gc.collect()
+
+    def slice_inference(self,
+                        raw_audio_path,
+                        spk,
+                        tran,
+                        slice_db,
+                        cluster_infer_ratio,
+                        auto_predict_f0,
+                        noice_scale,
+                        pad_seconds=0.5,
+                        clip_seconds=0,
+                        lg_num=0,
+                        lgr_num =0.75,
+                        f0_predictor='pm',
+                        enhancer_adaptive_key = 0,
+                        cr_threshold = 0.05,
+                        k_step = 100,
+                        use_spk_mix = False,
+                        second_encoding = False,
+                        loudness_envelope_adjustment = 1
+                        ):
+        if use_spk_mix:
+            if len(self.spk2id) == 1:
+                spk = self.spk2id.keys()[0]
+                use_spk_mix = False
+        wav_path = Path(raw_audio_path).with_suffix('.wav')
+        chunks = slicer.cut(wav_path, db_thresh=slice_db)
+        audio_data, audio_sr = slicer.chunks2audio(wav_path, chunks)
+        per_size = int(clip_seconds*audio_sr)
+        lg_size = int(lg_num*audio_sr)
+        lg_size_r = int(lg_size*lgr_num)
+        lg_size_c_l = (lg_size-lg_size_r)//2
+        lg_size_c_r = lg_size-lg_size_r-lg_size_c_l
+        lg = np.linspace(0,1,lg_size_r) if lg_size!=0 else 0
+
+        if use_spk_mix:
+            assert len(self.spk2id) == len(spk)
+            audio_length = 0
+            for (slice_tag, data) in audio_data:
+                aud_length = int(np.ceil(len(data) / audio_sr * self.target_sample))
+                if slice_tag:
+                    audio_length += aud_length // self.hop_size
+                    continue
+                if per_size != 0:
+                    datas = split_list_by_n(data, per_size,lg_size)
+                else:
+                    datas = [data]
+                for k,dat in enumerate(datas):
+                    pad_len = int(audio_sr * pad_seconds)
+                    per_length = int(np.ceil(len(dat) / audio_sr * self.target_sample))
+                    a_length = per_length + 2 * pad_len
+                    audio_length += a_length // self.hop_size
+            audio_length += len(audio_data)
+            spk_mix_tensor = torch.zeros(size=(len(spk), audio_length)).to(self.dev)
+            for i in range(len(spk)):
+                last_end = None
+                for mix in spk[i]:
+                    if mix[3]<0. or mix[2]<0.:
+                        raise RuntimeError("mix value must higer Than zero!")
+                    begin = int(audio_length * mix[0])
+                    end = int(audio_length * mix[1])
+                    length = end - begin
+                    if length<=0:                        
+                        raise RuntimeError("begin Must lower Than end!")
+                    step = (mix[3] - mix[2])/length
+                    if last_end is not None:
+                        if last_end != begin:
+                            raise RuntimeError("[i]EndTime Must Equal [i+1]BeginTime!")
+                    last_end = end
+                    if step == 0.:
+                        spk_mix_data = torch.zeros(length).to(self.dev) + mix[2]
+                    else:
+                        spk_mix_data = torch.arange(mix[2],mix[3],step).to(self.dev)
+                    if(len(spk_mix_data)<length):
+                        num_pad = length - len(spk_mix_data)
+                        spk_mix_data = torch.nn.functional.pad(spk_mix_data, [0, num_pad], mode="reflect").to(self.dev)
+                    spk_mix_tensor[i][begin:end] = spk_mix_data[:length]
+
+            spk_mix_ten = torch.sum(spk_mix_tensor,dim=0).unsqueeze(0).to(self.dev)
+            # spk_mix_tensor[0][spk_mix_ten<0.001] = 1.0
+            for i, x in enumerate(spk_mix_ten[0]):
+                if x == 0.0:
+                    spk_mix_ten[0][i] = 1.0
+                    spk_mix_tensor[:,i] = 1.0 / len(spk)
+            spk_mix_tensor = spk_mix_tensor / spk_mix_ten
+            if not ((torch.sum(spk_mix_tensor,dim=0) - 1.)<0.0001).all():
+                raise RuntimeError("sum(spk_mix_tensor) not equal 1")
+            spk = spk_mix_tensor
+
+        global_frame = 0
+        audio = []
+        for (slice_tag, data) in audio_data:
+            print(f'#=====segment start, {round(len(data) / audio_sr, 3)}s======')
+            # padd
+            length = int(np.ceil(len(data) / audio_sr * self.target_sample))
+            if slice_tag:
+                print('jump empty segment')
+                _audio = np.zeros(length)
+                audio.extend(list(pad_array(_audio, length)))
+                global_frame += length // self.hop_size
+                continue
+            if per_size != 0:
+                datas = split_list_by_n(data, per_size,lg_size)
+            else:
+                datas = [data]
+            for k,dat in enumerate(datas):
+                per_length = int(np.ceil(len(dat) / audio_sr * self.target_sample)) if clip_seconds!=0 else length
+                if clip_seconds!=0: print(f'###=====segment clip start, {round(len(dat) / audio_sr, 3)}s======')
+                # padd
+                pad_len = int(audio_sr * pad_seconds)
+                dat = np.concatenate([np.zeros([pad_len]), dat, np.zeros([pad_len])])
+                raw_path = io.BytesIO()
+                soundfile.write(raw_path, dat, audio_sr, format="wav")
+                raw_path.seek(0)
+                out_audio, out_sr, out_frame = self.infer(spk, tran, raw_path,
+                                                    cluster_infer_ratio=cluster_infer_ratio,
+                                                    auto_predict_f0=auto_predict_f0,
+                                                    noice_scale=noice_scale,
+                                                    f0_predictor = f0_predictor,
+                                                    enhancer_adaptive_key = enhancer_adaptive_key,
+                                                    cr_threshold = cr_threshold,
+                                                    k_step = k_step,
+                                                    frame = global_frame,
+                                                    spk_mix = use_spk_mix,
+                                                    second_encoding = second_encoding,
+                                                    loudness_envelope_adjustment = loudness_envelope_adjustment
+                                                    )
+                global_frame += out_frame
+                _audio = out_audio.cpu().numpy()
+                pad_len = int(self.target_sample * pad_seconds)
+                _audio = _audio[pad_len:-pad_len]
+                _audio = pad_array(_audio, per_length)
+                if lg_size!=0 and k!=0:
+                    lg1 = audio[-(lg_size_r+lg_size_c_r):-lg_size_c_r] if lgr_num != 1 else audio[-lg_size:]
+                    lg2 = _audio[lg_size_c_l:lg_size_c_l+lg_size_r]  if lgr_num != 1 else _audio[0:lg_size]
+                    lg_pre = lg1*(1-lg)+lg2*lg
+                    audio = audio[0:-(lg_size_r+lg_size_c_r)] if lgr_num != 1 else audio[0:-lg_size]
+                    audio.extend(lg_pre)
+                    _audio = _audio[lg_size_c_l+lg_size_r:] if lgr_num != 1 else _audio[lg_size:]
+                audio.extend(list(_audio))
+        return np.array(audio)
+
+class RealTimeVC:
+    def __init__(self):
+        self.last_chunk = None
+        self.last_o = None
+        self.chunk_len = 16000  # chunk length
+        self.pre_len = 3840  # cross fade length, multiples of 640
+
+    # Input and output are 1-dimensional numpy waveform arrays
+
+    def process(self, svc_model, speaker_id, f_pitch_change, input_wav_path,
+                cluster_infer_ratio=0,
+                auto_predict_f0=False,
+                noice_scale=0.4,
+                f0_filter=False):
+
+        import maad
+        audio, sr = torchaudio.load(input_wav_path)
+        audio = audio.cpu().numpy()[0]
+        temp_wav = io.BytesIO()
+        if self.last_chunk is None:
+            input_wav_path.seek(0)
+
+            audio, sr = svc_model.infer(speaker_id, f_pitch_change, input_wav_path,
+                                        cluster_infer_ratio=cluster_infer_ratio,
+                                        auto_predict_f0=auto_predict_f0,
+                                        noice_scale=noice_scale,
+                                        f0_filter=f0_filter)
+            
+            audio = audio.cpu().numpy()
+            self.last_chunk = audio[-self.pre_len:]
+            self.last_o = audio
+            return audio[-self.chunk_len:]
+        else:
+            audio = np.concatenate([self.last_chunk, audio])
+            soundfile.write(temp_wav, audio, sr, format="wav")
+            temp_wav.seek(0)
+
+            audio, sr = svc_model.infer(speaker_id, f_pitch_change, temp_wav,
+                                        cluster_infer_ratio=cluster_infer_ratio,
+                                        auto_predict_f0=auto_predict_f0,
+                                        noice_scale=noice_scale,
+                                        f0_filter=f0_filter)
+
+            audio = audio.cpu().numpy()
+            ret = maad.util.crossfade(self.last_o, audio, self.pre_len)
+            self.last_chunk = audio[-self.pre_len:]
+            self.last_o = audio
+            return ret[self.chunk_len:2 * self.chunk_len]
+            

inference/inference/infer_tool_grad.py

@@ -0,0 +1,160 @@
+import hashlib
+import json
+import logging
+import os
+import time
+from pathlib import Path
+import io
+import librosa
+import maad
+import numpy as np
+from inference import slicer
+import parselmouth
+import soundfile
+import torch
+import torchaudio
+
+from hubert import hubert_model
+import utils
+from models import SynthesizerTrn
+logging.getLogger('numba').setLevel(logging.WARNING)
+logging.getLogger('matplotlib').setLevel(logging.WARNING)
+
+def resize2d_f0(x, target_len):
+    source = np.array(x)
+    source[source < 0.001] = np.nan
+    target = np.interp(np.arange(0, len(source) * target_len, len(source)) / target_len, np.arange(0, len(source)),
+                       source)
+    res = np.nan_to_num(target)
+    return res
+
+def get_f0(x, p_len,f0_up_key=0):
+
+    time_step = 160 / 16000 * 1000
+    f0_min = 50
+    f0_max = 1100
+    f0_mel_min = 1127 * np.log(1 + f0_min / 700)
+    f0_mel_max = 1127 * np.log(1 + f0_max / 700)
+
+    f0 = parselmouth.Sound(x, 16000).to_pitch_ac(
+        time_step=time_step / 1000, voicing_threshold=0.6,
+        pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency']
+
+    pad_size=(p_len - len(f0) + 1) // 2
+    if(pad_size>0 or p_len - len(f0) - pad_size>0):
+        f0 = np.pad(f0,[[pad_size,p_len - len(f0) - pad_size]], mode='constant')
+
+    f0 *= pow(2, f0_up_key / 12)
+    f0_mel = 1127 * np.log(1 + f0 / 700)
+    f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (f0_mel_max - f0_mel_min) + 1
+    f0_mel[f0_mel <= 1] = 1
+    f0_mel[f0_mel > 255] = 255
+    f0_coarse = np.rint(f0_mel).astype(np.int)
+    return f0_coarse, f0
+
+def clean_pitch(input_pitch):
+    num_nan = np.sum(input_pitch == 1)
+    if num_nan / len(input_pitch) > 0.9:
+        input_pitch[input_pitch != 1] = 1
+    return input_pitch
+
+
+def plt_pitch(input_pitch):
+    input_pitch = input_pitch.astype(float)
+    input_pitch[input_pitch == 1] = np.nan
+    return input_pitch
+
+
+def f0_to_pitch(ff):
+    f0_pitch = 69 + 12 * np.log2(ff / 440)
+    return f0_pitch
+
+
+def fill_a_to_b(a, b):
+    if len(a) < len(b):
+        for _ in range(0, len(b) - len(a)):
+            a.append(a[0])
+
+
+def mkdir(paths: list):
+    for path in paths:
+        if not os.path.exists(path):
+            os.mkdir(path)
+
+
+class VitsSvc(object):
+    def __init__(self):
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.SVCVITS = None
+        self.hps = None
+        self.speakers = None
+        self.hubert_soft = utils.get_hubert_model()
+
+    def set_device(self, device):
+        self.device = torch.device(device)
+        self.hubert_soft.to(self.device)
+        if self.SVCVITS != None:
+            self.SVCVITS.to(self.device)
+
+    def loadCheckpoint(self, path):
+        self.hps = utils.get_hparams_from_file(f"checkpoints/{path}/config.json")
+        self.SVCVITS = SynthesizerTrn(
+            self.hps.data.filter_length // 2 + 1,
+            self.hps.train.segment_size // self.hps.data.hop_length,
+            **self.hps.model)
+        _ = utils.load_checkpoint(f"checkpoints/{path}/model.pth", self.SVCVITS, None)
+        _ = self.SVCVITS.eval().to(self.device)
+        self.speakers = self.hps.spk
+
+    def get_units(self, source, sr):
+        source = source.unsqueeze(0).to(self.device)
+        with torch.inference_mode():
+            units = self.hubert_soft.units(source)
+            return units
+
+
+    def get_unit_pitch(self, in_path, tran):
+        source, sr = torchaudio.load(in_path)
+        source = torchaudio.functional.resample(source, sr, 16000)
+        if len(source.shape) == 2 and source.shape[1] >= 2:
+            source = torch.mean(source, dim=0).unsqueeze(0)
+        soft = self.get_units(source, sr).squeeze(0).cpu().numpy()
+        f0_coarse, f0 = get_f0(source.cpu().numpy()[0], soft.shape[0]*2, tran)
+        return soft, f0
+
+    def infer(self, speaker_id, tran, raw_path):
+        speaker_id = self.speakers[speaker_id]
+        sid = torch.LongTensor([int(speaker_id)]).to(self.device).unsqueeze(0)
+        soft, pitch = self.get_unit_pitch(raw_path, tran)
+        f0 = torch.FloatTensor(clean_pitch(pitch)).unsqueeze(0).to(self.device)
+        stn_tst = torch.FloatTensor(soft)
+        with torch.no_grad():
+            x_tst = stn_tst.unsqueeze(0).to(self.device)
+            x_tst = torch.repeat_interleave(x_tst, repeats=2, dim=1).transpose(1, 2)
+            audio,_ = self.SVCVITS.infer(x_tst, f0=f0, g=sid)[0,0].data.float()
+        return audio, audio.shape[-1]
+
+    def inference(self,srcaudio,chara,tran,slice_db):
+        sampling_rate, audio = srcaudio
+        audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
+        if len(audio.shape) > 1:
+            audio = librosa.to_mono(audio.transpose(1, 0))
+        if sampling_rate != 16000:
+            audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000)
+        soundfile.write("tmpwav.wav", audio, 16000, format="wav")
+        chunks = slicer.cut("tmpwav.wav", db_thresh=slice_db)
+        audio_data, audio_sr = slicer.chunks2audio("tmpwav.wav", chunks)
+        audio = []
+        for (slice_tag, data) in audio_data:
+            length = int(np.ceil(len(data) / audio_sr * self.hps.data.sampling_rate))
+            raw_path = io.BytesIO()
+            soundfile.write(raw_path, data, audio_sr, format="wav")
+            raw_path.seek(0)
+            if slice_tag:
+                _audio = np.zeros(length)
+            else:
+                out_audio, out_sr = self.infer(chara, tran, raw_path)
+                _audio = out_audio.cpu().numpy()
+            audio.extend(list(_audio))
+        audio = (np.array(audio) * 32768.0).astype('int16')
+        return (self.hps.data.sampling_rate,audio)

inference/inference/slicer.py

@@ -0,0 +1,142 @@
+import librosa
+import torch
+import torchaudio
+
+
+class Slicer:
+    def __init__(self,
+                 sr: int,
+                 threshold: float = -40.,
+                 min_length: int = 5000,
+                 min_interval: int = 300,
+                 hop_size: int = 20,
+                 max_sil_kept: int = 5000):
+        if not min_length >= min_interval >= hop_size:
+            raise ValueError('The following condition must be satisfied: min_length >= min_interval >= hop_size')
+        if not max_sil_kept >= hop_size:
+            raise ValueError('The following condition must be satisfied: max_sil_kept >= hop_size')
+        min_interval = sr * min_interval / 1000
+        self.threshold = 10 ** (threshold / 20.)
+        self.hop_size = round(sr * hop_size / 1000)
+        self.win_size = min(round(min_interval), 4 * self.hop_size)
+        self.min_length = round(sr * min_length / 1000 / self.hop_size)
+        self.min_interval = round(min_interval / self.hop_size)
+        self.max_sil_kept = round(sr * max_sil_kept / 1000 / self.hop_size)
+
+    def _apply_slice(self, waveform, begin, end):
+        if len(waveform.shape) > 1:
+            return waveform[:, begin * self.hop_size: min(waveform.shape[1], end * self.hop_size)]
+        else:
+            return waveform[begin * self.hop_size: min(waveform.shape[0], end * self.hop_size)]
+
+    # @timeit
+    def slice(self, waveform):
+        if len(waveform.shape) > 1:
+            samples = librosa.to_mono(waveform)
+        else:
+            samples = waveform
+        if samples.shape[0] <= self.min_length:
+            return {"0": {"slice": False, "split_time": f"0,{len(waveform)}"}}
+        rms_list = librosa.feature.rms(y=samples, frame_length=self.win_size, hop_length=self.hop_size).squeeze(0)
+        sil_tags = []
+        silence_start = None
+        clip_start = 0
+        for i, rms in enumerate(rms_list):
+            # Keep looping while frame is silent.
+            if rms < self.threshold:
+                # Record start of silent frames.
+                if silence_start is None:
+                    silence_start = i
+                continue
+            # Keep looping while frame is not silent and silence start has not been recorded.
+            if silence_start is None:
+                continue
+            # Clear recorded silence start if interval is not enough or clip is too short
+            is_leading_silence = silence_start == 0 and i > self.max_sil_kept
+            need_slice_middle = i - silence_start >= self.min_interval and i - clip_start >= self.min_length
+            if not is_leading_silence and not need_slice_middle:
+                silence_start = None
+                continue
+            # Need slicing. Record the range of silent frames to be removed.
+            if i - silence_start <= self.max_sil_kept:
+                pos = rms_list[silence_start: i + 1].argmin() + silence_start
+                if silence_start == 0:
+                    sil_tags.append((0, pos))
+                else:
+                    sil_tags.append((pos, pos))
+                clip_start = pos
+            elif i - silence_start <= self.max_sil_kept * 2:
+                pos = rms_list[i - self.max_sil_kept: silence_start + self.max_sil_kept + 1].argmin()
+                pos += i - self.max_sil_kept
+                pos_l = rms_list[silence_start: silence_start + self.max_sil_kept + 1].argmin() + silence_start
+                pos_r = rms_list[i - self.max_sil_kept: i + 1].argmin() + i - self.max_sil_kept
+                if silence_start == 0:
+                    sil_tags.append((0, pos_r))
+                    clip_start = pos_r
+                else:
+                    sil_tags.append((min(pos_l, pos), max(pos_r, pos)))
+                    clip_start = max(pos_r, pos)
+            else:
+                pos_l = rms_list[silence_start: silence_start + self.max_sil_kept + 1].argmin() + silence_start
+                pos_r = rms_list[i - self.max_sil_kept: i + 1].argmin() + i - self.max_sil_kept
+                if silence_start == 0:
+                    sil_tags.append((0, pos_r))
+                else:
+                    sil_tags.append((pos_l, pos_r))
+                clip_start = pos_r
+            silence_start = None
+        # Deal with trailing silence.
+        total_frames = rms_list.shape[0]
+        if silence_start is not None and total_frames - silence_start >= self.min_interval:
+            silence_end = min(total_frames, silence_start + self.max_sil_kept)
+            pos = rms_list[silence_start: silence_end + 1].argmin() + silence_start
+            sil_tags.append((pos, total_frames + 1))
+        # Apply and return slices.
+        if len(sil_tags) == 0:
+            return {"0": {"slice": False, "split_time": f"0,{len(waveform)}"}}
+        else:
+            chunks = []
+            # 第一段静音并非从头开始，补上有声片段
+            if sil_tags[0][0]:
+                chunks.append(
+                    {"slice": False, "split_time": f"0,{min(waveform.shape[0], sil_tags[0][0] * self.hop_size)}"})
+            for i in range(0, len(sil_tags)):
+                # 标识有声片段（跳过第一段）
+                if i:
+                    chunks.append({"slice": False,
+                                   "split_time": f"{sil_tags[i - 1][1] * self.hop_size},{min(waveform.shape[0], sil_tags[i][0] * self.hop_size)}"})
+                # 标识所有静音片段
+                chunks.append({"slice": True,
+                               "split_time": f"{sil_tags[i][0] * self.hop_size},{min(waveform.shape[0], sil_tags[i][1] * self.hop_size)}"})
+            # 最后一段静音并非结尾，补上结尾片段
+            if sil_tags[-1][1] * self.hop_size < len(waveform):
+                chunks.append({"slice": False, "split_time": f"{sil_tags[-1][1] * self.hop_size},{len(waveform)}"})
+            chunk_dict = {}
+            for i in range(len(chunks)):
+                chunk_dict[str(i)] = chunks[i]
+            return chunk_dict
+
+
+def cut(audio_path, db_thresh=-30, min_len=5000):
+    audio, sr = librosa.load(audio_path, sr=None)
+    slicer = Slicer(
+        sr=sr,
+        threshold=db_thresh,
+        min_length=min_len
+    )
+    chunks = slicer.slice(audio)
+    return chunks
+
+
+def chunks2audio(audio_path, chunks):
+    chunks = dict(chunks)
+    audio, sr = torchaudio.load(audio_path)
+    if len(audio.shape) == 2 and audio.shape[1] >= 2:
+        audio = torch.mean(audio, dim=0).unsqueeze(0)
+    audio = audio.cpu().numpy()[0]
+    result = []
+    for k, v in chunks.items():
+        tag = v["split_time"].split(",")
+        if tag[0] != tag[1]:
+            result.append((v["slice"], audio[int(tag[0]):int(tag[1])]))
+    return result, sr

pretrain/meta.py

@@ -0,0 +1,31 @@
+def download_dict():
+    return {
+        "vec768l12": {
+            "url": "https://ibm.ent.box.com/shared/static/z1wgl1stco8ffooyatzdwsqn2psd9lrr",
+            "output": "./pretrain/checkpoint_best_legacy_500.pt"
+        },
+        "vec256l9": {
+            "url": "https://ibm.ent.box.com/shared/static/z1wgl1stco8ffooyatzdwsqn2psd9lrr",
+            "output": "./pretrain/checkpoint_best_legacy_500.pt"
+        },
+        "hubertsoft": {
+            "url": "https://github.com/bshall/hubert/releases/download/v0.1/hubert-soft-0d54a1f4.pt",
+            "output": "./pretrain/hubert-soft-0d54a1f4.pt"
+        },
+        "whisper-ppg": {
+            "url": "https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt",
+            "output": "./pretrain/medium.pt"
+        }
+    }
+
+
+def get_speech_encoder(config_path="configs/config.json"):
+    import json
+
+    with open(config_path, "r") as f:
+        data = f.read()
+        config = json.loads(data)
+        speech_encoder = config["model"]["speech_encoder"]
+        dict = download_dict()
+
+        return dict[speech_encoder]["url"], dict[speech_encoder]["output"]

pretrain/nsf_hifigan/config.json

@@ -0,0 +1,38 @@
+{
+    "resblock": "1",
+    "num_gpus": 4,
+    "batch_size": 10,
+    "learning_rate": 0.0002,
+    "adam_b1": 0.8,
+    "adam_b2": 0.99,
+    "lr_decay": 0.999,
+    "seed": 1234,
+
+    "upsample_rates":        [ 8, 8, 2, 2, 2],
+    "upsample_kernel_sizes": [16,16, 4, 4, 4],
+    "upsample_initial_channel": 512,
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "discriminator_periods": [3, 5, 7, 11, 17, 23, 37],
+
+    "segment_size": 16384,
+    "num_mels": 128,
+    "num_freq": 1025,
+    "n_fft"   : 2048,
+    "hop_size": 512,
+    "win_size": 2048,
+
+    "sampling_rate": 44100,
+
+    "fmin": 40,
+    "fmax": 16000,
+    "fmax_for_loss": null,
+
+    "num_workers": 16,
+
+    "dist_config": {
+        "dist_backend": "nccl",
+        "dist_url": "tcp://localhost:54321",
+        "world_size": 1
+    }
+}