import torch torch.manual_seed(0) torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True import nltk import time from Modules.diffusion.sampler import DiffusionSampler, ADPM2Sampler, KarrasSchedule nltk.download('punkt') from tortoise.utils.text import split_and_recombine_text import random random.seed(0) import numpy as np np.random.seed(0) import time import random import yaml from munch import Munch import numpy as np import torch from torch import nn import torch.nn.functional as F import torchaudio import librosa from nltk.tokenize import word_tokenize from models import * from utils import * from text_utils import TextCleaner import soundfile as sf textclenaer = TextCleaner() start_time = time.time() from resemble_enhance.enhancer.inference import denoise, enhance if torch.cuda.is_available(): device = "cuda" else: device = "cpu" ''' def _fn(path, solver, nfe, tau, denoising): if path is None: return None, None solver = solver.lower() nfe = int(nfe) lambd = 0.9 if denoising else 0.1 dwav, sr = torchaudio.load(path) dwav = dwav.mean(dim=0) wav1, new_sr = denoise(dwav, sr, device) wav2, new_sr = enhance(dwav, sr, device, nfe=nfe, solver=solver, lambd=lambd, tau=tau) wav1 = wav1.cpu().numpy() wav2 = wav2.cpu().numpy() sf.write('output_wav1.wav', wav1, new_sr) sf.write('output_wav2.wav', wav2, new_sr) return (new_sr, wav1), (new_sr, wav2) (new_sr, wav1), (new_sr, wav2) = _fn('/root/src/hf/videly/voices/huberman_clone.wav',"Midpoint",32,0.5,True) end_time = time.time() elapsed_time = end_time - start_time print(f"Loop took {elapsed_time} seconds to complete.") ''' start_time = time.time() to_mel = torchaudio.transforms.MelSpectrogram( n_mels=80, n_fft=2048, win_length=1200, hop_length=300) mean, std = -4, 4 def length_to_mask(lengths): mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths) mask = torch.gt(mask+1, lengths.unsqueeze(1)) return mask def preprocess(wave): wave_tensor = torch.from_numpy(wave).float() mel_tensor = to_mel(wave_tensor) mel_tensor = (torch.log(1e-5 + mel_tensor.unsqueeze(0)) - mean) / std return mel_tensor def compute_style(path): wave, sr = librosa.load(path, sr=24000) audio, index = librosa.effects.trim(wave, top_db=30) if sr != 24000: audio = librosa.resample(audio, sr, 24000) mel_tensor = preprocess(audio).to(device) with torch.no_grad(): ref_s = model.style_encoder(mel_tensor.unsqueeze(1)) ref_p = model.predictor_encoder(mel_tensor.unsqueeze(1)) return torch.cat([ref_s, ref_p], dim=1) device = 'cuda' if torch.cuda.is_available() else 'cpu' print(device) import phonemizer global_phonemizer = phonemizer.backend.EspeakBackend(language='en-us', preserve_punctuation=True, with_stress=True) config = yaml.safe_load(open("Configs/hg.yml")) print(config) # load pretrained ASR model ASR_config = config.get('ASR_config', False) ASR_path = config.get('ASR_path', False) text_aligner = load_ASR_models(ASR_path, ASR_config) # load pretrained F0 model F0_path = config.get('F0_path', False) pitch_extractor = load_F0_models(F0_path) # load BERT model from Utils.PLBERT.util import load_plbert BERT_path = config.get('PLBERT_dir', False) plbert = load_plbert(BERT_path) model_params = recursive_munch(config['model_params']) model = build_model(model_params, text_aligner, pitch_extractor, plbert) _ = [model[key].eval() for key in model] _ = [model[key].to(device) for key in model] params_whole = torch.load("Models/epochs_2nd_00020.pth", map_location='cpu') params = params_whole['net'] for key in model: if key in params: print('%s loaded' % key) try: model[key].load_state_dict(params[key]) except: from collections import OrderedDict state_dict = params[key] new_state_dict = OrderedDict() for k, v in state_dict.items(): name = k[7:] # remove `module.` new_state_dict[name] = v # load params model[key].load_state_dict(new_state_dict, strict=False) # except: # _load(params[key], model[key]) _ = [model[key].eval() for key in model] sampler = DiffusionSampler( model.diffusion.diffusion, sampler=ADPM2Sampler(), sigma_schedule=KarrasSchedule(sigma_min=0.0001, sigma_max=3.0, rho=9.0), # empirical parameters clamp=False ) def inference(text, ref_s, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1): text = text.strip() ps = global_phonemizer.phonemize([text]) ps = word_tokenize(ps[0]) ps = ' '.join(ps) tokens = textclenaer(ps) tokens.insert(0, 0) tokens = torch.LongTensor(tokens).to(device).unsqueeze(0) with torch.no_grad(): input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device) text_mask = length_to_mask(input_lengths).to(device) t_en = model.text_encoder(tokens, input_lengths, text_mask) bert_dur = model.bert(tokens, attention_mask=(~text_mask).int()) d_en = model.bert_encoder(bert_dur).transpose(-1, -2) s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device), embedding=bert_dur, embedding_scale=embedding_scale, features=ref_s, # reference from the same speaker as the embedding num_steps=diffusion_steps).squeeze(1) s = s_pred[:, 128:] ref = s_pred[:, :128] ref = alpha * ref + (1 - alpha) * ref_s[:, :128] s = beta * s + (1 - beta) * ref_s[:, 128:] d = model.predictor.text_encoder(d_en, s, input_lengths, text_mask) x, _ = model.predictor.lstm(d) duration = model.predictor.duration_proj(x) duration = torch.sigmoid(duration).sum(axis=-1) pred_dur = torch.round(duration.squeeze()).clamp(min=1) pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data)) c_frame = 0 for i in range(pred_aln_trg.size(0)): pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1 c_frame += int(pred_dur[i].data) # encode prosody en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device)) if model_params.decoder.type == "hifigan": asr_new = torch.zeros_like(en) asr_new[:, :, 0] = en[:, :, 0] asr_new[:, :, 1:] = en[:, :, 0:-1] en = asr_new F0_pred, N_pred = model.predictor.F0Ntrain(en, s) asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device)) if model_params.decoder.type == "hifigan": asr_new = torch.zeros_like(asr) asr_new[:, :, 0] = asr[:, :, 0] asr_new[:, :, 1:] = asr[:, :, 0:-1] asr = asr_new out = model.decoder(asr, F0_pred, N_pred, ref.squeeze().unsqueeze(0)) return out.squeeze().cpu().numpy()[..., :-50] def LFinference(text, s_prev, ref_s, alpha = 0.3, beta = 0.7, t = 0.7, diffusion_steps=5, embedding_scale=1): text = text.strip() ps = global_phonemizer.phonemize([text]) ps = word_tokenize(ps[0]) ps = ' '.join(ps) ps = ps.replace('``', '"') ps = ps.replace("''", '"') tokens = textclenaer(ps) tokens.insert(0, 0) tokens = torch.LongTensor(tokens).to(device).unsqueeze(0) with torch.no_grad(): input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device) text_mask = length_to_mask(input_lengths).to(device) t_en = model.text_encoder(tokens, input_lengths, text_mask) bert_dur = model.bert(tokens, attention_mask=(~text_mask).int()) d_en = model.bert_encoder(bert_dur).transpose(-1, -2) s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device), embedding=bert_dur, embedding_scale=embedding_scale, features=ref_s, # reference from the same speaker as the embedding num_steps=diffusion_steps).squeeze(1) if s_prev is not None: # convex combination of previous and current style s_pred = t * s_prev + (1 - t) * s_pred s = s_pred[:, 128:] ref = s_pred[:, :128] ref = alpha * ref + (1 - alpha) * ref_s[:, :128] s = beta * s + (1 - beta) * ref_s[:, 128:] s_pred = torch.cat([ref, s], dim=-1) d = model.predictor.text_encoder(d_en, s, input_lengths, text_mask) x, _ = model.predictor.lstm(d) duration = model.predictor.duration_proj(x) duration = torch.sigmoid(duration).sum(axis=-1) pred_dur = torch.round(duration.squeeze()).clamp(min=1) pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data)) c_frame = 0 for i in range(pred_aln_trg.size(0)): pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1 c_frame += int(pred_dur[i].data) # encode prosody en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device)) if model_params.decoder.type == "hifigan": asr_new = torch.zeros_like(en) asr_new[:, :, 0] = en[:, :, 0] asr_new[:, :, 1:] = en[:, :, 0:-1] en = asr_new F0_pred, N_pred = model.predictor.F0Ntrain(en, s) asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device)) if model_params.decoder.type == "hifigan": asr_new = torch.zeros_like(asr) asr_new[:, :, 0] = asr[:, :, 0] asr_new[:, :, 1:] = asr[:, :, 0:-1] asr = asr_new out = model.decoder(asr, F0_pred, N_pred, ref.squeeze().unsqueeze(0)) return out.squeeze().cpu().numpy()[..., :-100], s_pred # passage = ''' Psychology, a field as fascinating as it is complex, delves into the intricate workings of the human mind. At its core, it seeks to understand and explain how we think, feel, and behave. The journey into the realms of psychology embarks from the fundamental belief that human behavior is not random, but instead driven by internal and external factors, often intertwined in an intricate dance of cause and effect. The study of psychology branches out into various specializations, each focusing on different aspects of human behavior and mental processes. Clinical psychologists, for instance, explore the depths of mental health, working to diagnose, treat, and prevent mental disorders. Their work is pivotal in helping individuals navigate the often challenging waters of mental illness, offering therapies and interventions that can significantly improve the quality of life. On the other hand, developmental psychology provides insight into the growth and change that occur throughout a person's life. From the first words of a toddler to the wisdom of the elderly, developmental psychologists study how we evolve over time, shaping our understanding of the various stages of life. This specialization is crucial in understanding how early experiences influence behavior and personality in later years. Social psychology, another intriguing branch, examines how individuals are influenced by others. It uncovers the subtle yet powerful ways in which societal norms, group dynamics, and interpersonal relationships shape our actions and beliefs. Understanding these social factors is essential in addressing broader societal issues, from discrimination and prejudice to conflict and cooperation. ''' path = "output_wav2.wav" s_ref = compute_style(path) #sentences = passage.split('.') # simple split by comma #sentences = passage sentences = split_and_recombine_text(passage) wavs = [] s_prev = None for text in sentences: if text.strip() == "": continue text += '.' # add it back wav, s_prev = LFinference(text, s_prev, s_ref, alpha = 0, beta = 0.3, # make it more suitable for the text t = 0.7, diffusion_steps=10, embedding_scale=1) wavs.append(wav) audio_arrays = [] for wav_file in wavs: audio_arrays.append(wav_file) concatenated_audio = np.concatenate(audio_arrays) print('Synthesized: ') sf.write('huberman_clone_after_resemble.wav', concatenated_audio, 24000) end_time = time.time() elapsed_time = end_time - start_time print(f"Loop took {elapsed_time} seconds to complete.") def _fn(path, solver, nfe, tau, denoising): if path is None: return None, None solver = solver.lower() nfe = int(nfe) lambd = 0.9 if denoising else 0.1 dwav, sr = torchaudio.load(path) dwav = dwav.mean(dim=0) wav1, new_sr = denoise(dwav, sr, device) wav2, new_sr = enhance(dwav, sr, device, nfe=nfe, solver=solver, lambd=lambd, tau=tau) wav1 = wav1.cpu().numpy() wav2 = wav2.cpu().numpy() sf.write('enhanced.wav', wav2, new_sr) return (new_sr, wav1), (new_sr, wav2) (new_sr, wav1), (new_sr, wav2) = _fn('/root/src/hf/videly/huberman_clone_after_resemble.wav',"Midpoint",32,0.5,True)