TTS/tts/layers/tortoise/audio_utils.py

import os
from glob import glob
from typing import Dict, List

import librosa
import numpy as np
import torch
import torchaudio
from scipy.io.wavfile import read

from TTS.utils.audio.torch_transforms import TorchSTFT


def load_wav_to_torch(full_path):
    sampling_rate, data = read(full_path)
    if data.dtype == np.int32:
        norm_fix = 2**31
    elif data.dtype == np.int16:
        norm_fix = 2**15
    elif data.dtype == np.float16 or data.dtype == np.float32:
        norm_fix = 1.0
    else:
        raise NotImplementedError(f"Provided data dtype not supported: {data.dtype}")
    return (torch.FloatTensor(data.astype(np.float32)) / norm_fix, sampling_rate)


def check_audio(audio, audiopath: str):
    # Check some assumptions about audio range. This should be automatically fixed in load_wav_to_torch, but might not be in some edge cases, where we should squawk.
    # '2' is arbitrarily chosen since it seems like audio will often "overdrive" the [-1,1] bounds.
    if torch.any(audio > 2) or not torch.any(audio < 0):
        print(f"Error with {audiopath}. Max={audio.max()} min={audio.min()}")
    audio.clip_(-1, 1)


def read_audio_file(audiopath: str):
    if audiopath[-4:] == ".wav":
        audio, lsr = load_wav_to_torch(audiopath)
    elif audiopath[-4:] == ".mp3":
        audio, lsr = librosa.load(audiopath, sr=None)
        audio = torch.FloatTensor(audio)
    else:
        assert False, f"Unsupported audio format provided: {audiopath[-4:]}"

    # Remove any channel data.
    if len(audio.shape) > 1:
        if audio.shape[0] < 5:
            audio = audio[0]
        else:
            assert audio.shape[1] < 5
            audio = audio[:, 0]

    return audio, lsr


def load_required_audio(audiopath: str):
    audio, lsr = read_audio_file(audiopath)

    audios = [torchaudio.functional.resample(audio, lsr, sampling_rate) for sampling_rate in (22050, 24000)]
    for audio in audios:
        check_audio(audio, audiopath)

    return [audio.unsqueeze(0) for audio in audios]


def load_audio(audiopath, sampling_rate):
    audio, lsr = read_audio_file(audiopath)

    if lsr != sampling_rate:
        audio = torchaudio.functional.resample(audio, lsr, sampling_rate)
    check_audio(audio, audiopath)

    return audio.unsqueeze(0)


TACOTRON_MEL_MAX = 2.3143386840820312
TACOTRON_MEL_MIN = -11.512925148010254


def denormalize_tacotron_mel(norm_mel):
    return ((norm_mel + 1) / 2) * (TACOTRON_MEL_MAX - TACOTRON_MEL_MIN) + TACOTRON_MEL_MIN


def normalize_tacotron_mel(mel):
    return 2 * ((mel - TACOTRON_MEL_MIN) / (TACOTRON_MEL_MAX - TACOTRON_MEL_MIN)) - 1


def dynamic_range_compression(x, C=1, clip_val=1e-5):
    """
    PARAMS
    ------
    C: compression factor
    """
    return torch.log(torch.clamp(x, min=clip_val) * C)


def dynamic_range_decompression(x, C=1):
    """
    PARAMS
    ------
    C: compression factor used to compress
    """
    return torch.exp(x) / C


def get_voices(extra_voice_dirs: List[str] = []):
    dirs = extra_voice_dirs
    voices: Dict[str, List[str]] = {}
    for d in dirs:
        subs = os.listdir(d)
        for sub in subs:
            subj = os.path.join(d, sub)
            if os.path.isdir(subj):
                voices[sub] = list(glob(f"{subj}/*.wav")) + list(glob(f"{subj}/*.mp3")) + list(glob(f"{subj}/*.pth"))
    return voices


def load_voice(voice: str, extra_voice_dirs: List[str] = []):
    if voice == "random":
        return None, None

    voices = get_voices(extra_voice_dirs)
    paths = voices[voice]
    if len(paths) == 1 and paths[0].endswith(".pth"):
        return None, torch.load(paths[0])
    else:
        conds = []
        for cond_path in paths:
            c = load_required_audio(cond_path)
            conds.append(c)
        return conds, None


def load_voices(voices: List[str], extra_voice_dirs: List[str] = []):
    latents = []
    clips = []
    for voice in voices:
        if voice == "random":
            if len(voices) > 1:
                print("Cannot combine a random voice with a non-random voice. Just using a random voice.")
            return None, None
        clip, latent = load_voice(voice, extra_voice_dirs)
        if latent is None:
            assert (
                len(latents) == 0
            ), "Can only combine raw audio voices or latent voices, not both. Do it yourself if you want this."
            clips.extend(clip)
        elif clip is None:
            assert (
                len(clips) == 0
            ), "Can only combine raw audio voices or latent voices, not both. Do it yourself if you want this."
            latents.append(latent)
    if len(latents) == 0:
        return clips, None
    else:
        latents_0 = torch.stack([l[0] for l in latents], dim=0).mean(dim=0)
        latents_1 = torch.stack([l[1] for l in latents], dim=0).mean(dim=0)
        latents = (latents_0, latents_1)
        return None, latents


def wav_to_univnet_mel(wav, do_normalization=False, device="cuda"):
    stft = TorchSTFT(
        n_fft=1024,
        hop_length=256,
        win_length=1024,
        use_mel=True,
        n_mels=100,
        sample_rate=24000,
        mel_fmin=0,
        mel_fmax=12000,
    )
    stft = stft.to(device)
    mel = stft(wav)
    mel = dynamic_range_compression(mel)
    if do_normalization:
        mel = normalize_tacotron_mel(mel)
    return mel