app.py

from transformers import (
    AutoModelForSequenceClassification,  # For text emotion detection model
    AutoTokenizer, 
    pipeline,                           # For creating inference pipeline

)
from colorama import Fore, Style        # For colored console output
import pandas as pd                     # For data handling
import time
from datetime import datetime
import matplotlib.pyplot as plt
import gradio as gr
from fer import FER
import cv2

# Dictionaries to store emotion data over time
text_dataDict = {"Time": [], "Emotion": [], "Confidence Score": []}
face_dataDict = {"Time": [], "Emotion": [], "Confidence Score": []}

# Load model and tokenizer directly from HuggingFace
emotionDetectModel = AutoModelForSequenceClassification.from_pretrained("borisn70/bert-43-multilabel-emotion-detection")
tokenizer = AutoTokenizer.from_pretrained("borisn70/bert-43-multilabel-emotion-detection")  # Load tokenizer directly from model
pipe = pipeline(task="text-classification", model=emotionDetectModel, tokenizer=tokenizer)

face_emotion_detector = FER()

localFormat = "%Y-%m-%d %H:%M:%S" #this is how will print the timestamp: year-month-day hour-minutes-seconds (army time)
#currTime = datetime.now().astimezone().strftime(localFormat) this returns the time in the localFormat
#current_Time_Tuple = time.strptime(str(currTime), str(localFormat)) #creates a tuple that contains each part of the local format separate
#current_Time_In_Seconds = time.mktime(current_Time_Tuple) #converts the tuple into the number of seconds

def emotionAnalysis(message, face):
    """
    Main function that processes both text and facial emotions
    Args:
        message (str): User input text
    Returns:
        tuple: (str, plt) Contains the emotion results text and the updated plot
    """
    if (message.lower() == "quit"):
        return "Quitting...", displayResults()
    
    # Process text emotion
    result = pipe(message)
    text_emotion = result[0]["label"]
    text_score = result[0]["score"]
    words_timestamp = datetime.now().astimezone().strftime(localFormat)
    
    # Store text emotion data for plotting
    text_dataDict["Time"].append(words_timestamp)
    text_dataDict["Emotion"].append(text_emotion)
    text_dataDict["Confidence Score"].append(round(text_score, 2))
    
    # Capture and process facial emotion
    # Load the image using OpenCV (BGR format)
    img = cv2.imread(face)
    # Convert the image to RGB (FER uses RGB, OpenCV loads images in BGR format)
    if img is None:
        print("Error: Could not load the image. Check the image path.")
        return
    
    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    face_emotion, face_score = face_emotion_detector.top_emotion(img_rgb)
    face_timestamp = datetime.now().astimezone().strftime(localFormat)
    
    # Store facial emotion data for plotting
    face_dataDict["Time"].append(face_timestamp)
    face_dataDict["Emotion"].append(face_emotion)
    face_dataDict["Confidence Score"].append(face_score)
    
    # Return both the text result and the updated plot
    return f"Text: {text_emotion} | Face: {face_emotion}", displayResults()

def displayResults():
    """
    Creates and returns a matplotlib plot showing emotion trends over time
    Returns:
        matplotlib.pyplot: Plot object showing emotion analysis results
    """
    # Create a new figure with specified size
    plt.figure(figsize=(10, 6))
    
    # Set up plot labels and title
    plt.title("Emotions Detected Through Facial Expressions and Text Over Time")
    plt.xlabel("Time")
    plt.ylabel("Emotions")

    #plot facial emotions versus time where time is on the x-axis
    plt.plot(face_dataDict["Time"], face_dataDict["Emotion"], marker='o', linestyle='-', label="Facial Emotions")

    #plot facial emotions versus time where time is on the x-axis
    plt.plot(text_dataDict["Time"], text_dataDict["Emotion"], marker='o', linestyle='-', color='red', label="Text Emotions")

    #showing the graph and the legend
    plt.legend()
    plt.xticks(rotation=45)  # Rotate timestamps for better readability
    plt.tight_layout()       # Adjust layout to prevent label cutoff
    
    return plt

#with gr.Blocks as demo:
    #conesnt_radio = gr.Radio(["yes", "no"], label="This app uses your webcam to detect emotions from your face and reads your text inputs to determine emotions from your writing. Do you give consent? ")

# Create Gradio interface with consent notice in the description
interface = gr.Interface(
    fn=emotionAnalysis,
    inputs=[
        gr.Textbox(
            label="Enter your text",
            placeholder="Type your message here. Type 'quit' to see final results."
        ),
        gr.Image(label="Webcam Facial Expression", source="webcam")
    ],
    outputs=[
        gr.Text(label="Emotion Results"),
        gr.Plot(label="Emotion Timeline")
    ],
    title="Emotion Analysis from Text and Face",
    description="⚠️ This application will use your webcam to detect facial emotions. By using this app, you consent to webcam access. Type text and press Enter to analyze both text and facial emotions."
)

# Launch the interface
if __name__ == "__main__":
    interface.launch()