from transformers import ( AutoModelForSequenceClassification, # For text emotion detection model AutoTokenizer, pipeline, # For creating inference pipeline ) from colorama import Fore, Style # For colored console output import pandas as pd # For data handling import time from datetime import datetime import matplotlib.pyplot as plt import gradio as gr from fer import FER import cv2 # Dictionaries to store emotion data over time text_dataDict = {"Time": [], "Emotion": [], "Confidence Score": []} face_dataDict = {"Time": [], "Emotion": [], "Confidence Score": []} # Load model and tokenizer directly from HuggingFace emotionDetectModel = AutoModelForSequenceClassification.from_pretrained("borisn70/bert-43-multilabel-emotion-detection") tokenizer = AutoTokenizer.from_pretrained("borisn70/bert-43-multilabel-emotion-detection") # Load tokenizer directly from model pipe = pipeline(task="text-classification", model=emotionDetectModel, tokenizer=tokenizer) face_emotion_detector = FER() cap = cv2.VideoCapture(0) localFormat = "%Y-%m-%d %H:%M:%S" #this is how will print the timestamp: year-month-day hour-minutes-seconds (army time) #currTime = datetime.now().astimezone().strftime(localFormat) this returns the time in the localFormat #current_Time_Tuple = time.strptime(str(currTime), str(localFormat)) #creates a tuple that contains each part of the local format separate #current_Time_In_Seconds = time.mktime(current_Time_Tuple) #converts the tuple into the number of seconds def emotionAnalysis(message, face): """ Main function that processes both text and facial emotions Args: message (str): User input text face: Image input from Gradio interface, can be either: - numpy.ndarray: Direct webcam capture (RGB or BGR format) - str: File path to uploaded image Returns: tuple: (str, plt) Contains the emotion results text and the updated plot """ if (message.lower() == "quit"): return "Quitting...", displayResults() # Process text emotion result = pipe(message) text_emotion = result[0]["label"] text_score = result[0]["score"] words_timestamp = datetime.now().astimezone().strftime(localFormat) # Store text emotion data for plotting text_dataDict["Time"].append(words_timestamp) text_dataDict["Emotion"].append(text_emotion) text_dataDict["Confidence Score"].append(round(text_score, 2)) face_timestamp = datetime.now().astimezone().strftime(localFormat) if (face.any() == None): face_emotion = "Unreadable" face_dataDict["Time"].append(face_timestamp) face_dataDict["Emotion"].append(face_emotion) face_dataDict["Confidence Score"].append(0.0) else: # Capture and process facial emotion img_rgb = cv2.cvtColor(face, cv2.COLOR_BGR2RGB) face_emotion, face_score = face_emotion_detector.top_emotion(img_rgb) # Store facial emotion data for plotting face_dataDict["Time"].append(face_timestamp) face_dataDict["Emotion"].append(face_emotion) face_dataDict["Confidence Score"].append(face_score) # Return both the text result and the updated plot return f"Text: {text_emotion} | Face: {face_emotion}", displayResults() def displayResults(): """ Creates and returns a matplotlib plot showing emotion trends over time Returns: matplotlib.pyplot: Plot object showing emotion analysis results """ # Create a new figure with specified size plt.figure(figsize=(10, 6)) # Set up plot labels and title plt.title("Emotions Detected Through Facial Expressions and Text Over Time") plt.xlabel("Time") plt.ylabel("Emotions") #plot facial emotions versus time where time is on the x-axis plt.plot(face_dataDict["Time"], face_dataDict["Emotion"], marker='o', linestyle='-', label="Facial Emotions") #plot facial emotions versus time where time is on the x-axis plt.plot(text_dataDict["Time"], text_dataDict["Emotion"], marker='o', linestyle='-', color='red', label="Text Emotions") #showing the graph and the legend plt.legend() plt.xticks(rotation=45) # Rotate timestamps for better readability plt.tight_layout() # Adjust layout to prevent label cutoff return plt def process_webcam(img): """ Process webcam frame and draw emotion detection results Args: img: Input image from webcam Returns: numpy.ndarray: Image with emotion detection results drawn on it """ if img is None: return None try: # Convert to RGB for emotion detection #img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # Detect faces and emotions #result = face_emotion_detector.detect_emotions(img_rgb) return img except Exception as e: print(f"Error processing image: {str(e)}") return img ''' 2 rows, 2 columns column 1: inputs row 1, col 1 = user text input row 2, col 1 = camera live feed column 2: outputs row 1, col 2 = emotion results row 2, col 2 = plt graph ''' with gr.Blocks(title="Emotion Reader") as emotion_reader: gr.Markdown( """ # Emotion Analysis from Text and Face ⚠️ This application will use your webcam to detect facial emotions. By using this app, you consent to webcam access. Type text and press Enter to analyze both text and facial emotions. """ ) with gr.Row(): with gr.Column(): #user text input text_input = gr.Textbox( label="Type your message here. Type 'quit' to see final results.", placeholder="Enter text" ) with gr.Column(): #emotion results emotion_result = gr.Textbox(label="Emotion Results") with gr.Row(): with gr.Column(): #camera live feed input_img = gr.Image(label="Webcam Feed", sources="webcam") with gr.Column(): #plt graph output_img = gr.Image(label="Emotion Detection", visible=False) plot_output = gr.Plot(value=displayResults(), label="Emotion Timeline") # Stream webcam with emotion detection input_img.stream( process_webcam, inputs=input_img, outputs=output_img, time_limit=15, stream_every=0.1, concurrency_limit=30 ) # Process text input text_input.submit( emotionAnalysis, inputs=[text_input, output_img], outputs=[emotion_result, plot_output] ) # Launch the interface if __name__ == "__main__": emotion_reader.launch()