app.py

import gradio as gr
import cv2
import numpy as np
import torch
from torchvision import models, transforms
from torchvision.models.detection import FasterRCNN_ResNet50_FPN_Weights
from PIL import Image
import mediapipe as mp
from fer import FER  # Facial emotion recognition

# -----------------------------
# Configuration
# -----------------------------
# 1) Increase skip rate
SKIP_RATE = 15

# 2) Use GPU if available
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# 3) Desired input size for faster inference
DESIRED_SIZE = (640, 480)

# -----------------------------
# Global caches for overlay info and frame counters
# -----------------------------
posture_cache = {"landmarks": None, "text": "Initializing...", "counter": 0}
emotion_cache = {"text": "Initializing...", "counter": 0}
objects_cache = {"boxes": None, "text": "Initializing...", "counter": 0}
faces_cache = {"boxes": None, "text": "Initializing...", "counter": 0}

# -----------------------------
# Initialize Models and Helpers
# -----------------------------
mp_pose = mp.solutions.pose
pose = mp_pose.Pose()
mp_drawing = mp.solutions.drawing_utils

mp_face_detection = mp.solutions.face_detection
face_detection = mp_face_detection.FaceDetection(min_detection_confidence=0.5)

object_detection_model = models.detection.fasterrcnn_resnet50_fpn(
    weights=FasterRCNN_ResNet50_FPN_Weights.DEFAULT
)
object_detection_model.eval().to(device)  # Move model to GPU (if available)

obj_transform = transforms.Compose([transforms.ToTensor()])

# If the FER library supports GPU, it may pick it up automatically. 
# Some versions allow device specification, e.g. FER(mtcnn=True, device=device).
emotion_detector = FER(mtcnn=True)

# -----------------------------
# Overlay Drawing Functions
# -----------------------------
def draw_posture_overlay(raw_frame, landmarks):
    for (x, y) in landmarks:
        cv2.circle(raw_frame, (x, y), 4, (0, 255, 0), -1)
    return raw_frame

def draw_boxes_overlay(raw_frame, boxes, color):
    for (x1, y1, x2, y2) in boxes:
        cv2.rectangle(raw_frame, (x1, y1), (x2, y2), color, 2)
    return raw_frame

# -----------------------------
# Heavy (Synchronous) Detection Functions
# -----------------------------
def compute_posture_overlay(image):
    # Convert to BGR for MediaPipe
    frame_bgr = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
    h, w, _ = frame_bgr.shape

    # 2) Downscale before processing (optional for posture)
    frame_bgr_small = cv2.resize(frame_bgr, DESIRED_SIZE)
    small_h, small_w, _ = frame_bgr_small.shape

    frame_rgb_small = cv2.cvtColor(frame_bgr_small, cv2.COLOR_BGR2RGB)
    pose_results = pose.process(frame_rgb_small)

    # Scale landmarks back up to original size if needed
    if pose_results.pose_landmarks:
        landmarks = []
        for lm in pose_results.pose_landmarks.landmark:
            # Rescale from the smaller frame to the original size
            x = int(lm.x * small_w * (w / small_w))
            y = int(lm.y * small_h * (h / small_h))
            landmarks.append((x, y))
        text = "Posture detected"
    else:
        landmarks = []
        text = "No posture detected"

    return landmarks, text

def compute_emotion_overlay(image):
    # Convert to BGR
    frame_bgr = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
    # 2) Downscale
    frame_bgr_small = cv2.resize(frame_bgr, DESIRED_SIZE)
    frame_rgb_small = cv2.cvtColor(frame_bgr_small, cv2.COLOR_BGR2RGB)

    emotions = emotion_detector.detect_emotions(frame_rgb_small)
    if emotions:
        top_emotion, score = max(emotions[0]["emotions"].items(), key=lambda x: x[1])
        text = f"{top_emotion} ({score:.2f})"
    else:
        text = "No face detected"
    return text

def compute_objects_overlay(image):
    frame_bgr = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
    # 2) Downscale
    frame_bgr_small = cv2.resize(frame_bgr, DESIRED_SIZE)
    frame_rgb_small = cv2.cvtColor(frame_bgr_small, cv2.COLOR_BGR2RGB)

    image_pil = Image.fromarray(frame_rgb_small)
    img_tensor = obj_transform(image_pil).to(device)

    with torch.no_grad():
        detections = object_detection_model([img_tensor])[0]

    threshold = 0.8
    boxes = []
    for box, score in zip(detections["boxes"], detections["scores"]):
        if score > threshold:
            # box is in the scaled-down coordinates; 
            # you may want to scale them back to the original if needed
            boxes.append(tuple(box.int().cpu().numpy()))

    text = f"Detected {len(boxes)} object(s)" if boxes else "No objects detected"
    return boxes, text

def compute_faces_overlay(image):
    frame_bgr = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
    h, w, _ = frame_bgr.shape
    # 2) Downscale
    frame_bgr_small = cv2.resize(frame_bgr, DESIRED_SIZE)
    small_h, small_w, _ = frame_bgr_small.shape

    frame_rgb_small = cv2.cvtColor(frame_bgr_small, cv2.COLOR_BGR2RGB)
    face_results = face_detection.process(frame_rgb_small)

    boxes = []
    if face_results.detections:
        for detection in face_results.detections:
            bbox = detection.location_data.relative_bounding_box
            x = int(bbox.xmin * small_w)
            y = int(bbox.ymin * small_h)
            box_w = int(bbox.width * small_w)
            box_h = int(bbox.height * small_h)
            # Scale bounding box coords back to original if you need full resolution
            # E.g., x_original = int(x * (w / small_w)), etc.
            boxes.append((x, y, x + box_w, y + box_h))
        text = f"Detected {len(boxes)} face(s)"
    else:
        text = "No faces detected"
    return boxes, text

# -----------------------------
# Main Analysis Functions
# -----------------------------
def analyze_posture_current(image):
    global posture_cache
    posture_cache["counter"] += 1
    current_frame = np.array(image)

    if posture_cache["counter"] % SKIP_RATE == 0 or posture_cache["landmarks"] is None:
        landmarks, text = compute_posture_overlay(image)
        posture_cache["landmarks"] = landmarks
        posture_cache["text"] = text

    output = current_frame.copy()
    if posture_cache["landmarks"]:
        output = draw_posture_overlay(output, posture_cache["landmarks"])

    return output, f"Posture Analysis: {posture_cache['text']}"

def analyze_emotion_current(image):
    global emotion_cache
    emotion_cache["counter"] += 1
    current_frame = np.array(image)

    if emotion_cache["counter"] % SKIP_RATE == 0 or emotion_cache["text"] is None:
        text = compute_emotion_overlay(image)
        emotion_cache["text"] = text

    return current_frame, f"Emotion Analysis: {emotion_cache['text']}"

def analyze_objects_current(image):
    global objects_cache
    objects_cache["counter"] += 1
    current_frame = np.array(image)

    if objects_cache["counter"] % SKIP_RATE == 0 or objects_cache["boxes"] is None:
        boxes, text = compute_objects_overlay(image)
        objects_cache["boxes"] = boxes
        objects_cache["text"] = text

    output = current_frame.copy()
    if objects_cache["boxes"]:
        output = draw_boxes_overlay(output, objects_cache["boxes"], (255, 255, 0))

    return output, f"Object Detection: {objects_cache['text']}"

def analyze_faces_current(image):
    global faces_cache
    faces_cache["counter"] += 1
    current_frame = np.array(image)

    if faces_cache["counter"] % SKIP_RATE == 0 or faces_cache["boxes"] is None:
        boxes, text = compute_faces_overlay(image)
        faces_cache["boxes"] = boxes
        faces_cache["text"] = text

    output = current_frame.copy()
    if faces_cache["boxes"]:
        output = draw_boxes_overlay(output, faces_cache["boxes"], (0, 0, 255))

    return output, f"Face Detection: {faces_cache['text']}"

# -----------------------------
# Custom CSS
# -----------------------------
custom_css = """
@import url('https://fonts.googleapis.com/css2?family=Orbitron:wght@400;700&display=swap');
body {
    background-color: #0e0e0e;
    color: #ffffff;
    font-family: 'Orbitron', sans-serif;
    margin: 0;
    padding: 0;
}
.gradio-container {
    background: linear-gradient(135deg, #1e1e2f, #3e3e55);
    border-radius: 10px;
    padding: 20px;
    max-width: 1200px;
    margin: auto;
}
.gradio-title {
    font-size: 2.5em;
    color: #ffffff;
    text-align: center;
    margin-bottom: 0.2em;
}
.gradio-description {
    font-size: 1.2em;
    text-align: center;
    margin-bottom: 1em;
    color: #ffffff;
}
"""

# -----------------------------
# Create Individual Interfaces
# -----------------------------
posture_interface = gr.Interface(
    fn=analyze_posture_current,
    inputs=gr.Image(sources=["webcam"], streaming=True, label="Capture Your Posture"),
    outputs=[gr.Image(type="numpy", label="Annotated Output"), gr.Textbox(label="Posture Analysis")],
    title="Posture Analysis",
    description="Detects your posture using MediaPipe.",
    live=True  # Keep only this interface live to avoid multiple heavy computations
)

emotion_interface = gr.Interface(
    fn=analyze_emotion_current,
    inputs=gr.Image(sources=["webcam"], streaming=True, label="Capture Your Face"),
    outputs=[gr.Image(type="numpy", label="Annotated Output"), gr.Textbox(label="Emotion Analysis")],
    title="Emotion Analysis",
    description="Detects facial emotions using FER.",
    live=False  # Turn off streaming to reduce overhead
)

objects_interface = gr.Interface(
    fn=analyze_objects_current,
    inputs=gr.Image(sources=["webcam"], streaming=True, label="Capture the Scene"),
    outputs=[gr.Image(type="numpy", label="Annotated Output"), gr.Textbox(label="Object Detection")],
    title="Object Detection",
    description="Detects objects using a pretrained Faster R-CNN.",
    live=False
)

faces_interface = gr.Interface(
    fn=analyze_faces_current,
    inputs=gr.Image(sources=["webcam"], streaming=True, label="Capture Your Face"),
    outputs=[gr.Image(type="numpy", label="Annotated Output"), gr.Textbox(label="Face Detection")],
    title="Face Detection",
    description="Detects faces using MediaPipe.",
    live=False
)

# -----------------------------
# Create a Tabbed Interface
# -----------------------------
tabbed_interface = gr.TabbedInterface(
    interface_list=[posture_interface, emotion_interface, objects_interface, faces_interface],
    tab_names=["Posture", "Emotion", "Objects", "Faces"]
)

# -----------------------------
# Wrap in a Blocks Layout
# -----------------------------
demo = gr.Blocks(css=custom_css)
with demo:
    gr.Markdown("<h1 class='gradio-title'>Real-Time Multi-Analysis App</h1>")
    gr.Markdown(
        "<p class='gradio-description'>Experience a high-tech cinematic interface for real-time "
        "analysis of your posture, emotions, objects, and faces using your webcam.</p>"
    )
    tabbed_interface.render()

if __name__ == "__main__":
    demo.launch()