app.py

from transformers import pipeline
import matplotlib.pyplot as plt
import streamlit as st
from PIL import Image
import pandas as pd
import numpy as np
import cv2
import tempfile




pipe_yolos = pipeline("object-detection", model="hustvl/yolos-tiny")
pipe_emotions = pipeline("image-classification", model="dima806/facial_emotions_image_detection")
pipe_emotions_refined = pipeline("image-classification", model="felixwf/fine_tuned_face_emotion_model")


st.title("Online Teaching Effect Monitor")

file_name = st.file_uploader("Upload an image or a video")

if file_name is not None:
    if file_name.type.startswith('image'):
        # Process image
        face_image = Image.open(file_name)
        st.image(face_image)
        output = pipe_yolos(face_image)

        data = output
        # 过滤出所有标签为 "person" 的项
        persons = [item for item in data if item['label'] == 'person']
        
        # 打印结果
        print(persons)
        st.text(persons)
        st.subheader(f"Number of persons detected: {len(persons)}")
        
        # 假设有一张原始图片，加载图片并截取出每个 "person" 的部分
        original_image = face_image
        persons_image_list = []
        
        # 截取每个 "person" 的部分并保存
        for idx, person in enumerate(persons):
            box = person['box']
            cropped_image = original_image.crop((box['xmin'], box['ymin'], box['xmax'], box['ymax']))
            cropped_image.save(f'person_{idx}.jpg')
            cropped_image.show()
            persons_image_list.append(cropped_image)
        
        # Calculate the number of rows needed for 3 columns
        num_images = len(persons)
        num_cols = 8
        num_rows = (num_images + num_cols - 1) // num_cols  # Ceiling division

        # Create a new canvas to stitch all person images in a grid with 3 columns
        fig, axes = plt.subplots(num_rows, num_cols, figsize=(15, 2 * num_rows))

        # Flatten the axes array for easy iteration
        axes = axes.flatten()

        # Crop each "person" part and plot it on the grid
        for idx, person in enumerate(persons):
            box = person['box']
            cropped_image = original_image.crop((box['xmin'], box['ymin'], box['xmax'], box['ymax']))
            axes[idx].imshow(cropped_image)
            axes[idx].axis('off')
            axes[idx].set_title(f'Person {idx}')

        # Turn off any unused subplots
        for ax in axes[num_images:]:
            ax.axis('off')

        # 识别每个人的表情
        output_list_emotions = []
        output_list_emotions_refined = []
        
        for idx, face in enumerate(persons_image_list):
          print(f"processing {idx}")
          output = pipe_emotions(face)
          output_list_emotions.append(output[0])
          output = pipe_emotions_refined(face)
          output_list_emotions_refined.append(output[0])
        
        print(output_list_emotions)
        st.subheader("Emotions by model: dima806/facial_emotions_image_detection")
        st.text(output_list_emotions)
        print(output_list_emotions_refined)
        st.subheader("Actions by model: felixwf/fine_tuned_face_emotion_model")
        st.text(output_list_emotions_refined)

        
        # 统计各种标签的数量
        label_counts_emotions = {}
        label_counts_actions = {}
        for item in output_list_emotions:
            label = item['label']
            if label in label_counts_emotions:
                label_counts_emotions[label] += 1
            else:
                label_counts_emotions[label] = 1

        for item in output_list_emotions_refined:
            label = item['label']
            if label in label_counts_actions:
                label_counts_actions[label] += 1
            else:
                label_counts_actions[label] = 1

        # 绘制饼状图
        labels_emotions = list(label_counts_emotions.keys())
        sizes_emotions = list(label_counts_emotions.values())
        
        pie_fig_emotions, pie_ax_emotions = plt.subplots()
        pie_ax_emotions.pie(sizes_emotions, labels=labels_emotions, autopct='%1.1f%%', startangle=140)
        pie_ax_emotions.set_title('Distribution of Emotions')
        pie_ax_emotions.axis('equal')  # 确保饼状图为圆形

        labels_actions = list(label_counts_actions.keys())
        sizes_actions = list(label_counts_actions.values())
        
        pie_fig_actions, pie_ax_actions = plt.subplots()
        pie_ax_actions.pie(sizes_actions, labels=labels_actions, autopct='%1.1f%%', startangle=140)
        pie_ax_actions.set_title('Distribution of Actions')
        pie_ax_actions.axis('equal')  # 确保饼状图为圆形
        

        labels_refined = [item['label'] for item in output_list_emotions_refined]
        label_counts_refined = {label: labels_refined.count(label) for label in set(labels_refined)}

        bar_fig_actions, bar_ax_actions = plt.subplots()
        bar_ax_actions.bar(label_counts_refined.keys(), label_counts_refined.values())
        bar_ax_actions.set_title('Distribution of Actions')
        bar_ax_actions.set_xlabel('Emotions')
        bar_ax_actions.set_ylabel('Count')

        labels_emotions = [item['label'] for item in output_list_emotions]
        label_counts_emotions = {label: labels_emotions.count(label) for label in set(labels_emotions)}

        bar_fig_emotions, bar_ax_emotions = plt.subplots()
        bar_ax_emotions.bar(label_counts_emotions.keys(), label_counts_emotions.values())
        bar_ax_emotions.set_title('Distribution of Emotions')
        bar_ax_emotions.set_xlabel('Emotions')
        bar_ax_emotions.set_ylabel('Count')
        
        # plt.show()
        # Use Streamlit columns to display the images and pie chart side by side

        st.pyplot(fig)  # Display the stitched person images

        col1, col2 = st.columns(2)
        col1.pyplot(pie_fig_emotions)  # Display the pie chart
        col2.pyplot(bar_fig_emotions)  # Display the bar chart

        col1.pyplot(pie_fig_actions)  # Display the pie chart
        col2.pyplot(bar_fig_actions)  # Display the bar chart

    elif file_name.type.startswith('video'):
        # Save the uploaded video to a temporary file
        with tempfile.NamedTemporaryFile(delete=False) as temp_video_file:
            temp_video_file.write(file_name.read())
            temp_video_path = temp_video_file.name

        # Process video
        video = cv2.VideoCapture(temp_video_path)
        frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
        frame_rate = int(video.get(cv2.CAP_PROP_FPS))
        frame_interval = frame_rate  # Process one frame per second

        frame_emotions = []
        frame_emotions_refined = []
        for frame_idx in range(0, frame_count, frame_interval):
            video.set(cv2.CAP_PROP_POS_FRAMES, frame_idx)
            ret, frame = video.read()
            if not ret:
                break

            # Convert frame to PIL Image
            frame_image = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
            output = pipe_yolos(frame_image)

            data = output
            persons = [item for item in data if item['label'] == 'person']
            persons_image_list = []

            for person in persons:
                box = person['box']
                cropped_image = frame_image.crop((box['xmin'], box['ymin'], box['xmax'], box['ymax']))
                persons_image_list.append(cropped_image)

            # Recognize emotions for each person in the frame
            frame_emotion = []
            for face in persons_image_list:
                output = pipe_emotions(face)
                frame_emotion.append(output[0]['label'])
            frame_emotions.append(frame_emotion)
            
            frame_emotion_refined = []
            for face in persons_image_list:
                output = pipe_emotions_refined(face)
                frame_emotion_refined.append(output[0]['label'])
            frame_emotions_refined.append(frame_emotion_refined)

        # Plot number of persons detected over frames
        fig, ax = plt.subplots(figsize=(10, 5))
        ax.plot(range(len(frame_emotions)), [len(emotions) for emotions in frame_emotions], label='Number of Persons Detected')
        ax.set_xlabel('Frame')
        ax.set_ylabel('Number of Persons')
        ax.set_title('Number of Persons Detected Over Frames')
        ax.legend()

        st.pyplot(fig)

        # Plot emotions over frames, using the same frame index
        fig, ax = plt.subplots(figsize=(10, 5))
        for emotion in frame_emotions_refined[0]:
            ax.bar(range(len(frame_emotions_refined)), [emotion_counts[emotion] for emotion_counts in frame_emotions_refined], label=emotion)
        ax.set_xlabel('Frame')
        ax.set_ylabel('Emotion Count')
        ax.set_title('Emotion Distribution Over Frames')
        ax.legend()

        st.pyplot(fig)

        # Assuming frame_emotions_refined is a list of lists, where each sublist contains emotion labels for a frame
        fig, ax = plt.subplots(figsize=(10, 5))

        # Iterate over each frame's emotions
        for frame_idx, emotions in enumerate(frame_emotions_refined):
            # Count occurrences of each emotion in the current frame
            emotion_counts = {emotion: emotions.count(emotion) for emotion in set(emotions)}
            
            # Plot the emotion counts for the current frame
            ax.clear()
            ax.bar(emotion_counts.keys(), emotion_counts.values())
            ax.set_title(f"Frame {frame_idx + 1}")
            ax.set_xlabel('Emotions')
            ax.set_ylabel('Count')
            
            # Display the plot for the current frame
            st.pyplot(fig)

    else:
        st.error("Unsupported file type. Please upload an image or a video.")