app.py

import gradio as gr
import torch
import soundfile as sf
import spaces
import os
import numpy as np
import re
from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
from speechbrain.pretrained import EncoderClassifier
from datasets import load_dataset

device = "cuda" if torch.cuda.is_available() else "cpu"

def load_models_and_data():
    model_name = "microsoft/speecht5_tts"
    processor = SpeechT5Processor.from_pretrained(model_name)
    model = SpeechT5ForTextToSpeech.from_pretrained("emirhanbilgic/speecht5_finetuned_emirhan_tr").to(device)
    vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan").to(device)
    
    spk_model_name = "speechbrain/spkrec-xvect-voxceleb"
    speaker_model = EncoderClassifier.from_hparams(
        source=spk_model_name,
        run_opts={"device": device},
        savedir=os.path.join("/tmp", spk_model_name),
    )
    
    # Load a sample from a dataset for default embedding
    dataset = load_dataset("ylacombe/english_dialects ", split="train")
    example = dataset[304]
    
    return model, processor, vocoder, speaker_model, example

model, processor, vocoder, speaker_model, default_example = load_models_and_data()

# Choose the language dynamically (English or Regional Language)
model, processor, vocoder, speaker_model, default_example = load_models_and_data(language="en")

def create_speaker_embedding(waveform):
    with torch.no_grad():
        speaker_embeddings = speaker_model.encode_batch(torch.tensor(waveform).unsqueeze(0).to(device))
        speaker_embeddings = torch.nn.functional.normalize(speaker_embeddings, dim=2)
        speaker_embeddings = speaker_embeddings.squeeze()
    return speaker_embeddings

def prepare_default_embedding(example):
    audio = example["audio"]
    return create_speaker_embedding(audio["array"])

default_embedding = prepare_default_embedding(default_example)

# Text normalization updates for English technical speech
technical_replacements = [
    # Common technical replacements (examples)
    ("HTTP", "H T T P"),
    ("AI", "A I"),
    # Add more technical abbreviations as needed
]

def normalize_text(text, language="en"):
    text = text.lower()
    
    # Handle language-specific normalization
    if language == "en":
        # Replace technical terms or symbols
        for old, new in technical_replacements:
            text = text.replace(old, new)
    
    # For regional language, include character replacements like the Turkish example
    if language != "en":
        replacements = [
            # Character mappings for regional languages (like the Turkish example)
            # Add region/language-specific character normalization here
        ]
        for old, new in replacements:
            text = text.replace(old, new)
    
    # Remove punctuation or handle them contextually for technical speech
    text = re.sub(r'[^\w\s]', '', text)
    
    return text

@spaces.GPU(duration=60)
def text_to_speech(text, audio_file=None, language="en"):
    # Normalize the input text
    normalized_text = normalize_text(text, language=language)
    
    # Prepare the input for the model
    inputs = processor(text=normalized_text, return_tensors="pt").to(device)
    
    # Use the default speaker embedding
    speaker_embeddings = default_embedding
    
    # Generate speech
    with torch.no_grad():
        speech = model.generate_speech(inputs["input_ids"], speaker_embeddings.unsqueeze(0), vocoder=vocoder)
    
    speech_np = speech.cpu().numpy()

    return (16000, speech_np)
    
iface = gr.Interface(
    fn=text_to_speech,
    inputs=[
        gr.Textbox(label="Enter text to convert to speech"),
        gr.Dropdown(label="Language", choices=["English Technical", "Regional"], value="English Technical")
    ],
    outputs=[
        gr.Audio(label="Generated Speech", type="numpy")
    ],
    title="Fine-Tuned TTS for Technical English and Regional Languages",
    description="Enter text, choose the language, and listen to the generated speech."
)

iface.launch(share=True)