Spaces:
Runtime error
Runtime error
File size: 3,446 Bytes
d347764 21a68b4 d347764 7e3558a d347764 90bab96 d347764 2ee142b 6e2fa31 d347764 4dc6319 d347764 87dff83 2ee142b d347764 7e3558a d347764 c680c9d d347764 7e3558a b332b05 9a915aa e71ca8d d347764 7e3558a d7e755b 7e3558a d347764 48cb862 d347764 90bab96 d347764 f805e49 c6f1d54 f805e49 c737803 d347764 226ec3a d347764 f805e49 d347764 c737803 3946ba6 c737803 70a5583 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 |
import gradio as gr
import numpy as np
import torch
from datasets import load_dataset
import librosa
from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor, pipeline
from transformers import VitsModel, VitsTokenizer
target_dtype = np.int16
max_range = np.iinfo(target_dtype).max
device = "cuda:0" if torch.cuda.is_available() else "cpu"
# load speech translation checkpoint
asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-base", device=device)
# load text-to-speech checkpoint and speaker embeddings
processor = SpeechT5Processor.from_pretrained("sanchit-gandhi/speecht5_tts_vox_nl")
model = SpeechT5ForTextToSpeech.from_pretrained("sanchit-gandhi/speecht5_tts_vox_nl").to(device)
vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan").to(device)
embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
model_mms = VitsModel.from_pretrained("facebook/mms-tts-nld")
tokenizer_mms = VitsTokenizer.from_pretrained("facebook/mms-tts-nld")
def translate(audio):
outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"language": "nl","task": "transcribe"})
return outputs["text"]
def synthesise_speechT5(text):
inputs = processor(text=text, padding='max_length', truncation=True,max_length=600,return_tensors="pt")
print(inputs)
speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device),vocoder=vocoder)
return speech.cpu()
def synthesise(text):
inputs = tokenizer_mms(text, return_tensors="pt")
input_ids = inputs["input_ids"]
with torch.no_grad():
outputs = model_mms(input_ids)
speech = outputs["waveform"]
return speech
def speech_to_speech_translation(audio):
sampling_rate = 16000
data_array,samplerate = librosa.load(audio)
data_16 = librosa.resample(data_array, orig_sr=samplerate, target_sr=sampling_rate)
translated_text = translate(data_16)
synthesised_speech = synthesise(translated_text)
synthesised_speech = (synthesised_speech.numpy() * max_range).astype(np.int16)
return sampling_rate, synthesised_speech
title = "Cascaded STST"
description = """
Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in English. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and Microsoft's
[SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech:
"""
demo = gr.Blocks()
mic_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="microphone", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
title=title,
description=description,
)
file_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="upload", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
examples=[["./example.wav"]],
title=title,
description=description,
)
with demo:
gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
demo.launch()
|