Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import numpy as np | |
| import torch | |
| from datasets import load_dataset | |
| from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor, pipeline | |
| device = "cuda:0" if torch.cuda.is_available() else "cpu" | |
| # Load Whisper large-v2 model for multilingual speech translation | |
| asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-large-v2", device=device) | |
| # Load MMS TTS model for multilingual text-to-speech (using German model as base) | |
| processor = SpeechT5Processor.from_pretrained("facebook/s2t-medium-mustc-multilingual-st") | |
| model = SpeechT5ForTextToSpeech.from_pretrained("facebook/s2t-medium-mustc-multilingual-st").to(device) | |
| vocoder = SpeechT5HifiGan.from_pretrained("facebook/s2t-medium-mustc-multilingual-st").to(device) | |
| # Define supported languages (adjust based on the languages supported by the model) | |
| LANGUAGES = { | |
| "German": "deu", "English": "eng", "French": "fra", "Spanish": "spa", | |
| "Italian": "ita", "Portuguese": "por", "Polish": "pol", "Turkish": "tur" | |
| } | |
| def translate(audio, source_lang, target_lang): | |
| outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={ | |
| "task": "transcribe", | |
| "language": source_lang, | |
| }) | |
| transcription = outputs["text"] | |
| # Use Whisper for translation | |
| translation = asr_pipe(transcription, max_new_tokens=256, generate_kwargs={ | |
| "task": "translate", | |
| "language": target_lang, | |
| })["text"] | |
| return translation | |
| def synthesise(text, target_lang): | |
| inputs = processor(text=text, return_tensors="pt") | |
| speech = model.generate_speech(inputs["input_ids"].to(device), vocoder=vocoder, language=LANGUAGES[target_lang]) | |
| return speech.cpu() | |
| def speech_to_speech_translation(audio, source_lang, target_lang): | |
| translated_text = translate(audio, LANGUAGES[source_lang], LANGUAGES[target_lang]) | |
| synthesised_speech = synthesise(translated_text, target_lang) | |
| synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16) | |
| return 16000, synthesised_speech | |
| title = "Multilingual Speech-to-Speech Translation" | |
| description = """ | |
| Demo for multilingual speech-to-speech translation (STST), mapping from source speech in any supported language to target speech in any other supported language. | |
| """ | |
| demo = gr.Blocks() | |
| with demo: | |
| gr.Markdown(f"# {title}") | |
| gr.Markdown(description) | |
| with gr.Row(): | |
| source_lang = gr.Dropdown(choices=list(LANGUAGES.keys()), label="Source Language") | |
| target_lang = gr.Dropdown(choices=list(LANGUAGES.keys()), label="Target Language") | |
| with gr.Tabs(): | |
| with gr.TabItem("Microphone"): | |
| mic_input = gr.Audio(source="microphone", type="filepath") | |
| mic_output = gr.Audio(label="Generated Speech", type="numpy") | |
| mic_button = gr.Button("Translate") | |
| with gr.TabItem("Audio File"): | |
| file_input = gr.Audio(source="upload", type="filepath") | |
| file_output = gr.Audio(label="Generated Speech", type="numpy") | |
| file_button = gr.Button("Translate") | |
| mic_button.click( | |
| speech_to_speech_translation, | |
| inputs=[mic_input, source_lang, target_lang], | |
| outputs=mic_output | |
| ) | |
| file_button.click( | |
| speech_to_speech_translation, | |
| inputs=[file_input, source_lang, target_lang], | |
| outputs=file_output | |
| ) | |
| demo.launch() |