from transformers import ( AutoModelForSequenceClassification, # For text emotion detection model AutoTokenizer, pipeline, # For creating inference pipeline ) from datetime import datetime import matplotlib.pyplot as plt import gradio as gr from fer import FER import cv2 import os # Dictionaries to store emotion data over time text_dataDict = {"Time": [], "Emotion": [], "Confidence Score": []} face_dataDict = {"Time": [], "Emotion": [], "Confidence Score": []} # List of temporary files to clean up temp_files = [] # Load model and tokenizer directly from HuggingFace emotionDetectModel = AutoModelForSequenceClassification.from_pretrained("borisn70/bert-43-multilabel-emotion-detection") tokenizer = AutoTokenizer.from_pretrained("borisn70/bert-43-multilabel-emotion-detection") # Load tokenizer directly from model pipe = pipeline(task="text-classification", model=emotionDetectModel, tokenizer=tokenizer) face_emotion_detector = FER() localFormat = "%Y-%m-%d %H:%M:%S" #this is how will print the timestamp: year-month-day hour-minutes-seconds (army time) #currTime = datetime.now().astimezone().strftime(localFormat) this returns the time in the localFormat #current_Time_Tuple = time.strptime(str(currTime), str(localFormat)) #creates a tuple that contains each part of the local format separate #current_Time_In_Seconds = time.mktime(current_Time_Tuple) #converts the tuple into the number of seconds def emotionAnalysis(message, face): """ Main function that processes both text and facial emotions Args: message (str): User input text face: Image input from Gradio interface, can be either: - numpy.ndarray: Direct webcam capture (RGB or BGR format) - str: File path to uploaded image Returns: tuple: (str, plt, Gradio Download Button Object) Contains the emotion results text and the updated plot """ if (message.lower() == "finish"): graph = displayResults() filename = "Emotion_Timeline.png" graph.savefig(filename) temp_files.append(filename) download = gr.DownloadButton(label="Download Emotion Timeline", value=filename, visible=True) return "Quitting...", graph, download # Process text emotion result = pipe(message) text_emotion = result[0]["label"] text_score = result[0]["score"] words_timestamp = datetime.now().astimezone().strftime(localFormat) # Store text emotion data for plotting text_dataDict["Time"].append(words_timestamp) text_dataDict["Emotion"].append(text_emotion) text_dataDict["Confidence Score"].append(round(text_score, 2)) face_timestamp = datetime.now().astimezone().strftime(localFormat) # Initialize with default values face_emotion = "No image" # Default value face_score = 0.0 if face is not None: try: img_rgb = cv2.cvtColor(face, cv2.COLOR_BGR2RGB) result = face_emotion_detector.top_emotion(img_rgb) print(result) if result[0] is not None: # Only update if we got a valid result face_emotion, face_score = result else: face_emotion = "No face detected" face_score = 0.0 except Exception as e: face_emotion = f"Error processing image: {str(e)}" face_score = 0.0 # Store facial emotion data for plotting face_dataDict["Time"].append(face_timestamp) face_dataDict["Emotion"].append(face_emotion) # Now face_emotion will always be a string face_dataDict["Confidence Score"].append(face_score) data = displayResults() file_name = "unfinishedPLT.png" data.savefig(file_name) temp_files.append(file_name) dL = gr.DownloadButton(label="Download Emotion Timeline", value=file_name, visible=False) # Return both the text result and the updated plot return f"Text: {text_emotion} | Face: {face_emotion}", data, dL def displayResults(): """ Creates and returns a matplotlib plot showing emotion trends over time Returns: matplotlib.pyplot: Plot object showing emotion analysis results """ # Create a new figure with specified size plt.figure(figsize=(10, 6)) # Set up plot labels and title plt.title("Emotions Detected Through Facial Expressions and Text Over Time") plt.xlabel("Time") plt.ylabel("Emotions") #plot facial emotions versus time where time is on the x-axis plt.plot(face_dataDict["Time"], face_dataDict["Emotion"], marker='o', linestyle='-', label="Facial Emotions") #plot facial emotions versus time where time is on the x-axis plt.plot(text_dataDict["Time"], text_dataDict["Emotion"], marker='o', linestyle='-', color='red', label="Text Emotions") #showing the graph and the legend plt.legend() plt.xticks(rotation=45) # Rotate timestamps for better readability plt.tight_layout() # Adjust layout to prevent label cutoff return plt def process_webcam(img): """ Process webcam frame and draw emotion detection results """ if img is None: return None try: return img except Exception as e: print(f"Error processing image: {str(e)}") return img def cleanUp_Files(): """ Removes temporary plot files created during the application's runtime """ for file in temp_files: try: if os.path.exists(file): os.remove(file) print(f"Cleaned up {file}") except Exception as e: print(f"Error cleaning up {file}: {str(e)}") ''' 2 rows, 2 columns column 1: inputs row 1, col 1 = user text input row 2, col 1 = camera live feed column 2: outputs row 1, col 2 = emotion results row 2, col 2 = plt graph ''' with gr.Blocks(title="Emotion Reader", theme=gr.themes.Ocean()) as emotion_reader: gr.Markdown( """ # Emotion Analysis from Text and Face ⚠️ This application will use your webcam to detect facial emotions. By using this app, you consent to webcam access. Type text and press Enter to analyze both text and facial emotions. Steps to use the app: 1. Turn on the camera clicking where it says "Click to Access Webcam" and Allow access 2. Click "Record" (the dropdown arrow is for if you want to change your camera) 3. Type a sentence into the text input box 4. Press "Enter" to see your results 5. In the "Emotion Results" box, you will see something like "Text: (emotion) | Face: (emotion) " and the timeline will update 6. You can press "Stop" to turn off the camera or type "finish" as your message to be able to download your results """ ) with gr.Row(): with gr.Column(): #user text input text_input = gr.Textbox( label="Type your thoughts here. Type 'finish' to see final results.", placeholder="Enter text" ) examples = gr.Examples( examples=[ "I am feeling happy today!", "I am feeling sad today.", "I wish I could go on vacation." ], inputs=text_input ) with gr.Column(): #emotion results emotion_result = gr.Textbox(label="Emotion Results") download_button = gr.DownloadButton(label="Download Emotion Timeline", visible=False) with gr.Row(): with gr.Column(): #camera live feed input_img = gr.Image(label="Webcam Feed", sources="webcam") with gr.Column(): #plt graph output_img = gr.Image(label="Emotion Detection", visible=False) plot_output = gr.Plot(value=displayResults(), label="Emotion Timeline") # Stream webcam with emotion detection input_img.stream( process_webcam, inputs=input_img, outputs=output_img, time_limit=15, stream_every=0.1, concurrency_limit=30 ) # Process text input text_input.submit( emotionAnalysis, inputs=[text_input, output_img], outputs=[emotion_result, plot_output, download_button] ) #cleanup files download_button.click( cleanUp_Files ) # Launch the interface if __name__ == "__main__": emotion_reader.launch()