main.py

from google.colab import drive
drive.mount('/content/drive')

"""Install Dependencies"""

pip install transformers librosa torch soundfile numba numpy TTS datasets gradio protobuf==3.20.3

"""Emotion Detection (Using Text Dataset)

"""

!pip install --upgrade numpy tensorflow transformers TTS

!pip freeze > requirements.txt

from transformers import pipeline

# Load pre-trained model for emotion detection
emotion_classifier = pipeline("text-classification", model="bhadresh-savani/distilbert-base-uncased-emotion")

def detect_emotion(text):
    result = emotion_classifier(text)
    emotion = result[0]['label']
    confidence = result[0]['score']
    return emotion, confidence

# Example usage
text = "I am feeling excited today!"
emotion, confidence = detect_emotion(text)
print(f"Detected Emotion: {emotion}, Confidence: {confidence}")

"""Emotion-Aware TTS (Using Tacotron 2 or Similar)"""

import torch
import librosa
import numpy as np
from TTS.api import TTS  # Using Coqui TTS for simplicity

# Load TTS model and vocoder automatically during initialization
tts_model = TTS(model_name="tts_models/en/ljspeech/tacotron2-DDC")


def generate_emotional_speech(text, emotion):
    # Map emotion to voice modulation parameters (pitch, speed)
    emotion_settings = {
    "neutral": {"pitch": 1.0, "speed": 1.0},     # Baseline conversational tone
    "joy": {"pitch": 1.3, "speed": 1.2},         # Upbeat and energetic
    "sadness": {"pitch": 0.8, "speed": 0.9},     # Subdued, slow tone
    "anger": {"pitch": 1.6, "speed": 1.4},       # Intense and sharp
    "fear": {"pitch": 1.2, "speed": 0.95},       # Tense and slightly slow
    "surprise": {"pitch": 1.5, "speed": 1.3},    # Excitement with high pitch and fast speech
    "disgust": {"pitch": 0.9, "speed": 0.95},    # Low and deliberate
    "shame": {"pitch": 0.8, "speed": 0.85},      # Quiet, subdued tone

}


    # Retrieve pitch and speed based on detected emotion
    settings = emotion_settings.get(emotion, {"pitch": 1.0, "speed": 1.0})
    # Generate speech with the TTS model
    # Instead of directly passing speed and pitch to tts_to_file,
    # We adjust the text to simulate the effect. This is a temporary solution.
    # You might need to fine-tune these adjustments or consider a different TTS library
    # with better control over speech parameters.
    adjusted_text = text
    if settings['speed'] > 1.0:
        adjusted_text = adjusted_text.replace(" ", ".")  # Simulate faster speech
    elif settings['speed'] < 1.0:
        adjusted_text = adjusted_text.replace(" ", "...")  # Simulate slower speech

    # Explicitly specify the output path
    audio_path = "output.wav"  # Or any desired filename
    tts_model.tts_to_file(text=adjusted_text, file_path=audio_path)  # Pass file_path argument
    return audio_path

# Example usage
emotion = "happy"
output_audio = generate_emotional_speech("Welcome to the smart library!", emotion)
print(f"Generated Speech Saved At: {output_audio}")

"""Integrating the Workflow"""

from IPython.display import Audio, display

def emotion_aware_tts_pipeline(text):
    emotion, confidence = detect_emotion(text)
    print(f"Emotion Detected: {emotion} with Confidence: {confidence:.2f}")

    audio_path = generate_emotional_speech(text, emotion)
    print(f"Audio Generated: {audio_path}")

    # Display and play the audio
    display(Audio(audio_path, autoplay=True))

# Example usage
emotion_aware_tts_pipeline("I can’t stooop smiiiling, everything feels perrrfect!")

"""Fine-tuning the Emotion Detection Model"""

import os
os.environ["WANDB_DISABLED"] = "true"

from google.colab import drive
from transformers import Trainer, TrainingArguments, AutoModelForSequenceClassification, AutoTokenizer
from datasets import load_dataset

# Load dataset
dataset = load_dataset('/content/drive/MyDrive/Emotion_Model')  #path_to_your_dataset

# Preprocess data
tokenizer = AutoTokenizer.from_pretrained("bhadresh-savani/distilbert-base-uncased-emotion")

# Define a function to map emotion labels to integers
def map_emotion_to_int(example):
    # Assuming your dataset has an 'emotion' column with string labels
    # Replace this with your actual emotion labels and their corresponding integers
    # *Change 'emotion' to the actual column name in your dataset*
    emotion_mapping = {
        "neutral": 0,
        "joy": 1,
        "sad": 2,
        "anger": 3,
        "fear": 4,
        "surprise": 5,
        "disgust": 6,
        "shame": 7,
    }
    # Assuming your emotion column is named 'label'
    # example['label'] = emotion_mapping[example['emotion']] # Create a new 'label' column with integer values
    example['label'] = emotion_mapping.get(example['label'], -1) # If the label is not in the emotion mapping then we set it to -1. We can later filter these examples out
    return example


def preprocess_data(example):
    return tokenizer(example['text'], truncation=True, padding=True, max_length=512) # Added max_length for consistency


# Apply emotion mapping before tokenization
dataset = dataset.map(map_emotion_to_int, batched=False)
# *Keep the 'label' column for training. Only remove 'text'*
# Filter out examples with labels not in emotion_mapping (-1)
dataset = dataset.filter(lambda example: example['label'] != -1) # Filter out examples with label -1
tokenized_dataset = dataset.map(preprocess_data, batched=True, remove_columns=['text'])

# Load model
# model = AutoModelForSequenceClassification.from_pretrained("bhadresh-savani/distilbert-base-uncased-emotion", num_labels=8)

# Load model with ignore_mismatched_sizes=True
model = AutoModelForSequenceClassification.from_pretrained(
    "bhadresh-savani/distilbert-base-uncased-emotion",
    num_labels=8,
    ignore_mismatched_sizes=True
)

# Training arguments
training_args = TrainingArguments(
    output_dir="./results",    # Directory for model checkpoints and logs
    evaluation_strategy="epoch",  # Evaluate after every epoch
    learning_rate=5e-5,  # Start with 5e-5 (slightly higher than default 2e-5)
    per_device_train_batch_size=16,  # Use 16 for balance between memory usage and training speed
    gradient_accumulation_steps=4,  # Accumulate gradients to simulate larger batch size
    num_train_epochs=5,  # Train for 4-5 epochs (typically enough for fine-tuning)
    weight_decay=0.01,  # Regularization to avoid overfitting
    save_strategy="epoch",  # Save checkpoints after each epoch
    logging_dir="./logs",  # Directory for logging
    logging_steps=100,  # Log every 100 steps
    warmup_steps=500,  # Gradual learning rate increase for the first 500 steps
    save_total_limit=3,  # Keep only the last 3 checkpoints
    fp16=True,  # Enable mixed precision for faster training if GPU supports it
    load_best_model_at_end=True,  # Load the best model at the end of training
    metric_for_best_model="eval_loss",  # Use evaluation loss to select the best model
    greater_is_better=False,  # Lower loss is better
)


# Train model
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_dataset['train'],
    eval_dataset=tokenized_dataset['validation'],
    tokenizer=tokenizer,
)

trainer.train()

# Save the model and tokenizer to Google Drive
model_save_path = "/content/drive/My Drive/emotion_detection_model1"
tokenizer_save_path = "/content/drive/My Drive/emotion_detection_model1"

# Save the fine-tuned model
model.save_pretrained(model_save_path)
tokenizer.save_pretrained(tokenizer_save_path)

print("Model and tokenizer saved to Google Drive.")

"""Reload the Fine-Tuned Model"""

from transformers import AutoModelForSequenceClassification, AutoTokenizer

# Mount Google Drive
from google.colab import drive
drive.mount('/content/drive')

# Path to the saved model and tokenizer
model_save_path = "/content/drive/My Drive/emotion_detection_model"
tokenizer_save_path = "/content/drive/My Drive/emotion_detection_model"

# Load the fine-tuned model and tokenizer
model = AutoModelForSequenceClassification.from_pretrained(model_save_path)
tokenizer = AutoTokenizer.from_pretrained(tokenizer_save_path)

print("Fine-tuned model and tokenizer loaded successfully.")

"""Test the Reloaded Model"""

from transformers import pipeline

# Create a text classification pipeline with the loaded model
emotion_classifier = pipeline("text-classification", model=model, tokenizer=tokenizer)

# Test with a sample text
text = "I feel so upset today!"
result = emotion_classifier(text)
print(result)

"""Fine-tuning the TTS System"""

from TTS.api import TTS
from TTS.utils.audio import AudioProcessor
from TTS.tts.models.tacotron2 import Tacotron2
import torch

# Load pre-trained model
#model = Tacotron2.load_model("tts_models/en/ljspeech/tacotron2-DDC")
tts = TTS(model_name="tts_models/en/ljspeech/tacotron2-DDC") # Use TTS for model loading

# Access the Tacotron2 model from the TTS object
model = tts.synthesizer.tts_model

# Fine-tuning parameters
model.config.dataset_path = "/content/drive/MyDrive/RAVDESS"
model.config.num_epochs = 10

# Train
model.train()

# Define the save path on Google Drive
save_path = "/content/drive/My Drive/fine_tuned_tacotron2.pth"

# Save the model's state dictionary using torch.save
torch.save(model.state_dict(), save_path)
    

"""Set up the Gradio interface"""

import gradio as gr
from transformers import pipeline
from TTS.api import TTS

# Load pre-trained emotion detection model
emotion_classifier = pipeline("text-classification", model="bhadresh-savani/distilbert-base-uncased-emotion")

# Load TTS model
tts_model = TTS(model_name="tts_models/en/ljspeech/tacotron2-DDC")

# Emotion-specific settings for pitch and speed
emotion_settings = {
    "neutral": {"pitch": 1.0, "speed": 1.0},
    "joy": {"pitch": 1.3, "speed": 1.2},
    "sadness": {"pitch": 0.8, "speed": 0.9},
    "anger": {"pitch": 1.6, "speed": 1.4},
    "fear": {"pitch": 1.2, "speed": 0.95},
    "surprise": {"pitch": 1.5, "speed": 1.3},
    "disgust": {"pitch": 0.9, "speed": 0.95},
    "shame": {"pitch": 0.8, "speed": 0.85},
}

import librosa
import soundfile as sf

def adjust_audio_speed(audio_path, speed_factor):
    y, sr = librosa.load(audio_path)
    y_speeded = librosa.effects.time_stretch(y, speed_factor)
    sf.write(audio_path, y_speeded, sr)

def adjust_audio_pitch(audio_path, pitch_factor):
    y, sr = librosa.load(audio_path)
    y_shifted = librosa.effects.pitch_shift(y, sr, n_steps=pitch_factor)
    sf.write(audio_path, y_shifted, sr)

def emotion_aware_tts_pipeline(input_text=None, file_input=None):
    try:
        # Get text from input or file
        if file_input:
            with open(file_input.name, 'r') as file:
                input_text = file.read()

        if input_text:
            # Detect emotion
            emotion_data = emotion_classifier(input_text)[0]
            emotion = emotion_data['label']
            confidence = emotion_data['score']

            # Adjust pitch and speed
            settings = emotion_settings.get(emotion.lower(), {"pitch": 1.0, "speed": 1.0})
            pitch = settings["pitch"]
            speed = settings["speed"]

            # Generate audio
            audio_path = "output.wav"
            tts_model.tts_to_file(text=input_text, file_path=audio_path)

            # Adjust pitch and speed using librosa
            if pitch != 1.0:
                adjust_audio_pitch(audio_path, pitch)
            if speed != 1.0:
                adjust_audio_speed(audio_path, speed)

            return f"Detected Emotion: {emotion} (Confidence: {confidence:.2f})", audio_path
        else:
            return "Please provide input text or file", None
    except Exception as e:
        return f"Error: {str(e)}", None

  

# Define Gradio interface
iface = gr.Interface(
    fn=emotion_aware_tts_pipeline,
    inputs=[
        gr.Textbox(label="Input Text", placeholder="Enter text here"),
        gr.File(label="Upload a Text File")
    ],
    outputs=[
        gr.Textbox(label="Detected Emotion"),
        gr.Audio(label="Generated Audio")
    ],
    title="Emotion-Aware Text-to-Speech",
    description="Input text or upload a text file to detect the emotion and generate audio with emotion-aware modulation."
)

# Launch Gradio interface
iface.launch()