app.py

#!/usr/bin/env python3

import gradio as gr
import numpy as np
import torch
import json
import io
import soundfile as sf
from PIL import Image
import matplotlib
import joblib
from sklearn.decomposition import PCA
from collections import OrderedDict

matplotlib.use("Agg")  # Use non-interactive backend
import matplotlib.pyplot as plt

from text2speech import tts_randomized, parse_speed, tts_with_style_vector

# Constants and Paths
VOICES_JSON_PATH = "voices.json"
PCA_MODEL_PATH = "pca_model.pkl"
ANNOTATED_FEATURES_PATH = "annotated_features.npy"
VECTOR_DIMENSION = 256
ANNOTATED_FEATURES_NAMES = ["Gender", "Tone", "Quality", "Enunciation", "Pace", "Style"]
ANNOTATED_FEATURES_INFO = [
    "Male | Female",
    "High | Low",
    "Noisy | Clean",
    "Clear | Unclear",
    "Rapid | Slow",
    "Colloquial | Formal",
]

# Load PCA model and annotated features
try:
    pca = joblib.load(PCA_MODEL_PATH)
    print("PCA model loaded successfully.")
except FileNotFoundError:
    print(f"Error: PCA model file '{PCA_MODEL_PATH}' not found.")
    pca = None

try:
    annotated_features = np.load(ANNOTATED_FEATURES_PATH)
    print("Annotated features loaded successfully.")
except FileNotFoundError:
    print(f"Error: Annotated features file '{ANNOTATED_FEATURES_PATH}' not found.")
    annotated_features = None

# Utility Functions


def load_voices_json():
    """Load the voices.json file."""
    try:
        with open(VOICES_JSON_PATH, "r") as f:
            return json.load(f, object_pairs_hook=OrderedDict)
    except FileNotFoundError:
        print(f"Warning: {VOICES_JSON_PATH} not found. Creating a new one.")
        return OrderedDict()
    except json.JSONDecodeError:
        print(f"Warning: {VOICES_JSON_PATH} is not valid JSON.")
        return OrderedDict()


def save_voices_json(data, path=VOICES_JSON_PATH):
    """Save to voices.json."""
    with open(path, "w") as f:
        json.dump(data, f, indent=2)
    print(f"Voices saved to '{path}'.")


def update_sliders(voice_name):
    """
    Update slider values based on the selected predefined voice using reverse PCA.

    Args:
        voice_name (str): The name of the selected voice.

    Returns:
        list: A list of PCA component values to set the sliders.
    """
    if not voice_name:
        # Return default slider values (e.g., zeros) if no voice is selected
        return [0.0] * len(ANNOTATED_FEATURES_NAMES)

    voices_data = load_voices_json()
    if voice_name not in voices_data:
        print(f"Voice '{voice_name}' not found in {VOICES_JSON_PATH}.")
        return [0.0] * len(ANNOTATED_FEATURES_NAMES)

    style_vector = np.array(voices_data[voice_name], dtype=np.float32).reshape(1, -1)

    if pca is None:
        print("PCA model is not loaded.")
        return [0.0] * len(ANNOTATED_FEATURES_NAMES)

    try:
        # Transform the style vector into PCA component values
        pca_components = pca.transform(style_vector)[0]
        return pca_components.tolist()
    except Exception as e:
        print(f"Error transforming style vector to PCA components: {e}")
        return [0.0] * len(ANNOTATED_FEATURES_NAMES)


def generate_audio_with_voice(text, voice_key, speed_val):
    """
    Generate audio using the style vector of the selected predefined voice.

    Args:
        text (str): The text to synthesize.
        voice_key (str): The name of the selected voice.
        speed_val (float): The speed multiplier.

    Returns:
        tuple: (audio_tuple, style_vector)
    """
    try:
        # Load voices data
        voices_data = load_voices_json()

        if voice_key not in voices_data:
            print(f"Voice '{voice_key}' not found in {VOICES_JSON_PATH}.")
            return None, None, "Selected voice not found."

        # Retrieve the style vector for the selected voice
        style_vector = np.array(voices_data[voice_key], dtype=np.float32).reshape(1, -1)
        print(f"Selected Voice: {voice_key}")
        print(f"Style Vector (First 6): {style_vector[0][:6]}")

        # Convert to torch tensor
        style_vec_torch = torch.from_numpy(style_vector).float()

        # Generate audio using the TTS model
        audio_np = tts_with_style_vector(
            text,
            style_vec=style_vec_torch,
            speed=speed_val,
            alpha=0.3,
            beta=0.7,
            diffusion_steps=7,
            embedding_scale=1.0,
        )

        if audio_np is None:
            print("Audio generation failed.")
            return None, None, "Audio generation failed."

        # Prepare audio for Gradio
        sr = 24000  # Adjust based on your actual sampling rate
        audio_tuple = (sr, audio_np)

        # Return audio, image, and style vector
        return audio_tuple, style_vector.tolist()

    except Exception as e:
        print(f"Error in generate_audio_with_voice: {e}")
        return None, None, "An error occurred during audio generation."


def build_modified_vector(voice_key, top6_values):
    """Build a modified style vector by updating top 6 PCA components."""
    voices_data = load_voices_json()
    if voice_key not in voices_data:
        print(f"Voice '{voice_key}' not found in {VOICES_JSON_PATH}.")
        return None

    arr = np.array(voices_data[voice_key], dtype=np.float32).squeeze()
    if arr.ndim != 1 or arr.shape[0] != VECTOR_DIMENSION:
        print(f"Voice '{voice_key}' has invalid shape {arr.shape}. Expected (256,).")
        return None

    try:
        # Reconstruct the style vector using inverse PCA
        pca_components = np.array(top6_values).reshape(1, -1)
        reconstructed_vec = pca.inverse_transform(pca_components)[0]
        return reconstructed_vec
    except Exception as e:
        print(f"Error reconstructing style vector: {e}")
        return None


def reconstruct_style_vector(pca_components):
    """
    Reconstruct the 256-dimensional style vector from PCA components.
    """
    if pca is None:
        print("PCA model is not loaded.")
        return None
    try:
        return pca.inverse_transform([pca_components])[0]
    except Exception as e:
        print(f"Error during inverse PCA transform: {e}")
        return None


def generate_custom_audio(text, voice_key, randomize, speed_str, *slider_values):
    """
    Generate audio and produce a matplotlib plot of the style vector.
    Returns:
      - audio tuple (sr, np_array) for Gradio's Audio
      - a PIL Image representing the style vector plot
      - the final style vector as a list for State
    """
    try:
        speed_val = parse_speed(speed_str)
        print(f"Parsed speed: {speed_val}")

        if randomize:
            # Generate randomized style vector
            audio_np, random_style_vec = tts_randomized(text, speed=speed_val)
            if random_style_vec is None:
                print("Failed to generate randomized style vector.")
                return None, None, None
            # Ensure the style vector is flat
            final_vec = (
                random_style_vec.numpy().flatten()
                if isinstance(random_style_vec, torch.Tensor)
                else np.array(random_style_vec).flatten()
            )
            print("Randomized Style Vector (First 6):", final_vec[:6])
        else:
            # Reconstruct the style vector from slider values using inverse PCA
            reconstructed_vec = build_modified_vector(voice_key, slider_values)
            if reconstructed_vec is None:
                print(
                    "No reconstructed vector could be constructed, skipping audio generation."
                )
                return None, None, None

            # Convert to torch tensor
            style_vec_torch = torch.from_numpy(reconstructed_vec).float().unsqueeze(0)

            # Generate audio with the reconstructed style vector
            audio_np = tts_with_style_vector(
                text,
                style_vec=style_vec_torch,
                speed=speed_val,
                alpha=0.3,
                beta=0.7,
                diffusion_steps=7,
                embedding_scale=1.0,
            )
            final_vec = reconstructed_vec
            print("Reconstructed Style Vector (First 6):", final_vec[:6])

        if audio_np is None:
            print("Audio generation failed.")
            return None, None, None

        # Prepare audio for Gradio
        sr = 24000  # Adjust based on your actual sampling rate
        audio_tuple = (sr, audio_np)

        # Return audio, image, and style vector
        return audio_tuple, final_vec.tolist()

    except Exception as e:
        print(f"Error generating audio and style plot: {e}")
        return None, None, None


def save_style_to_json(style_data, style_name):
    """Saves the provided style_data (list of floats) into voices.json under style_name."""
    if not style_name.strip():
        return "Please enter a new style name before saving."

    voices_data = load_voices_json()
    if style_name in voices_data:
        return (
            f"Style name '{style_name}' already exists. Please choose a different name."
        )

    # Ensure the style_data has the correct length
    if len(style_data) != VECTOR_DIMENSION:
        return f"Style vector length mismatch. Expected {VECTOR_DIMENSION}, got {len(style_data)}."

    # Save the style vector
    voices_data[style_name] = style_data
    save_voices_json(voices_data)
    return f"Saved style as '{style_name}' in {VOICES_JSON_PATH}."


# Gradio Interface Functions


def rearrange_voices(new_order):
    """Rearrange the voices based on the new_order list."""
    voices_data = load_voices_json()
    new_order_list = [name.strip() for name in new_order.split(",")]
    if not all(name in voices_data for name in new_order_list):
        return "Error: New order contains invalid voice names.", list(
            voices_data.keys()
        )
    ordered_data = OrderedDict()
    for name in new_order_list:
        ordered_data[name] = voices_data[name]
    save_voices_json(ordered_data)
    print(f"Voices rearranged: {list(ordered_data.keys())}")
    return "Voices rearranged successfully.", list(ordered_data.keys())


def delete_voice(selected):
    """Delete voices from the voices.json."""
    if not selected:
        return "No voices selected for deletion.", list(load_voices_json().keys())
    voices_data = load_voices_json()
    for voice_name in selected:
        if voice_name in voices_data:
            del voices_data[voice_name]
            print(f"Voice '{voice_name}' deleted.")
    save_voices_json(voices_data)
    return "Deleted selected voices successfully.", list(voices_data.keys())


def upload_new_voices(uploaded_file):
    """Upload new voices from a JSON file."""
    if uploaded_file is None:
        return "No file uploaded.", list(load_voices_json().keys())
    try:
        uploaded_data = json.load(uploaded_file)
        if not isinstance(uploaded_data, dict):
            return "Invalid JSON format. Expected a dictionary of voices.", list(
                load_voices_json().keys()
            )
        voices_data = load_voices_json()
        voices_data.update(uploaded_data)
        save_voices_json(voices_data)
        print(f"Voices uploaded: {list(uploaded_data.keys())}")
        return "Voices uploaded successfully.", list(voices_data.keys())
    except json.JSONDecodeError:
        return "Uploaded file is not valid JSON.", list(load_voices_json().keys())


# Create Gradio Interface with Tabs


def create_combined_interface():
    voices_data = load_voices_json()
    voice_choices = list(voices_data.keys())
    default_voice = voice_choices[0] if voice_choices else None

    css = """
    h4 {
        text-align: center;
        display:block;
    }
    """

    def refresh_voices():
        """Refresh the voices by reloading the JSON."""
        new_choices = list(load_voices_json().keys())
        print(f"Voices refreshed: {new_choices}")
        return gr.Dropdown(choices=new_choices)

    with gr.Blocks(theme=gr.themes.Ocean(), css=css) as demo:
        gr.Markdown("# StyleTTS2 Studio - Build custom voices")

        # ----------- Text-to-Speech Tab -----------
        with gr.Tab("Text-to-Speech"):
            gr.Markdown("### Generate Speech with Predefined Voices")

            with gr.Column():
                text_input = gr.Textbox(
                    label="Text to Synthesize",
                    value="Hello world from the Gradio + TTS pipeline!",
                    lines=3,
                )
                voice_dropdown = gr.Dropdown(
                    choices=voice_choices,
                    label="Select Base Voice",
                    value=default_voice,
                    interactive=True,
                )
                speed_slider = gr.Slider(
                    minimum=50,
                    maximum=200,
                    step=1,
                    label="Speed (%)",
                    value=100,
                )
                with gr.Row():
                    generate_btn = gr.Button("Generate Audio")

            audio_output = gr.Audio(label="Synthesized Audio")

            # Generate button functionality
            def on_generate_tts(text, voice, speed):
                if not voice:
                    return None, "No voice selected."
                speed_val = speed / 100  # Convert percentage to multiplier
                audio, style_vector = generate_audio_with_voice(text, voice, speed_val)
                if audio is None:
                    return None, style_vector  # style_vector contains the error message
                return audio, "Audio generated successfully."

            generate_btn.click(
                fn=on_generate_tts,
                inputs=[text_input, voice_dropdown, speed_slider],
                outputs=[audio_output, gr.Textbox(label="Status", visible=False)],
            )

        # ----------- Voice Studio Tab -----------
        with gr.Tab("Voice Studio"):
            gr.Markdown("### Customize and Create New Voices")

            with gr.Column():
                text_input_studio = gr.Textbox(
                    label="Text to Synthesize",
                    value="Customize your voice here!",
                    lines=3,
                )
                voice_dropdown_studio = gr.Dropdown(
                    choices=voice_choices,
                    label="Select Base Voice",
                    value=default_voice,
                )
                speed_slider_studio = gr.Slider(
                    minimum=50,
                    maximum=200,
                    step=1,
                    label="Speed (%)",
                    value=100,
                )
                # Sliders for PCA components (6 sliders)
                pca_sliders = [
                    gr.Slider(
                        minimum=-2.0,
                        maximum=2.0,
                        value=0.0,
                        step=0.1,
                        label=feature,
                    )
                    for feature in ANNOTATED_FEATURES_NAMES
                ]

            generate_btn_studio = gr.Button("Generate Customized Audio")
            audio_output_studio = gr.Audio(label="Customized Synthesized Audio")
            new_style_name = gr.Textbox(label="New Style Name", value="")
            save_btn_studio = gr.Button("Save Customized Voice")
            status_text = gr.Textbox(label="Status", visible=True)

            # State to hold the last style vector
            style_vector_state_studio = gr.State()

            # Generate button functionality
            def on_generate_studio(text, voice, speed, *pca_values):
                if not voice:
                    return None, "No voice selected.", None
                speed_val = speed / 100  # Convert percentage to multiplier
                result = generate_custom_audio(
                    text, voice, False, speed_val, *pca_values
                )
                if result is None:
                    return None, "Failed to generate audio.", None
                audio_tuple, style_vector = result
                style_vector_state_studio.value = style_vector
                return audio_tuple, "Audio generated successfully.", style_vector

            generate_btn_studio.click(
                fn=on_generate_studio,
                inputs=[text_input_studio, voice_dropdown_studio, speed_slider_studio]
                + pca_sliders,
                outputs=[audio_output_studio, status_text, style_vector_state_studio],
            )

            # Save button functionality
            def on_save_style_studio(style_vector, style_name):
                if not style_name:
                    return "Please enter a name for the new voice!"
                result = save_style_to_json(style_vector, style_name)
                new_choices = list(load_voices_json().keys())
                # Return multiple values to update both dropdowns and show status
                return (
                    gr.Dropdown(choices=new_choices),  # Update first dropdown
                    gr.Dropdown(choices=new_choices),  # Update studio dropdown
                    result,  # Status message
                )

            save_btn_studio.click(
                fn=on_save_style_studio,
                inputs=[style_vector_state_studio, new_style_name],
                outputs=[voice_dropdown, voice_dropdown_studio, status_text],
            )

            # Add callback to update sliders when a voice is selected
            voice_dropdown_studio.change(
                fn=update_sliders,
                inputs=voice_dropdown_studio,
                outputs=pca_sliders,
            )

        gr.Markdown(
            "#### Based on [StyleTTS2](https://github.com/yl4579/StyleTTS2) and [artificial StyleTTS2](https://huggingface.co/dkounadis/artificial-styletts2/tree/main)"
        )

    return demo


if __name__ == "__main__":
    try:
        interface = create_combined_interface()
        interface.launch(share=False)
    except Exception as e:
        print(f"An error occurred while launching the interface: {e}")