app.py

# import logging
# import queue
# from pathlib import Path
# from typing import List, NamedTuple
# import mediapipe as mp
# import av
# import cv2
# import numpy as np
# import streamlit as st
# from streamlit_webrtc import WebRtcMode, webrtc_streamer
# from sample_utils.turn import get_ice_servers
# from cvzone.HandTrackingModule import HandDetector
# from cvzone.SelfiSegmentationModule import SelfiSegmentation
# import time
# import os

# logger = logging.getLogger(__name__)

# st.title("Interactive Virtual Keyboard with Twilio Integration")
# st.info("Use your webcam to interact with the virtual keyboard via hand gestures.")

# class Button:
#     def __init__(self, pos, text, size=[100, 100]):
#         self.pos = pos
#         self.size = size
#         self.text = text

# # Initialize components
# detector = HandDetector(maxHands=1, detectionCon=0.8)
# # segmentor = SelfiSegmentation()
# # keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
# #         ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
# #         ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]

# # listImg = os.listdir('model/street')
# # imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg]
# # indexImg = 0


# # # Function to process the video frame from the webcam
# # def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state):
# #     # Convert the frame to a numpy array (BGR format)
# #     image = frame.to_ndarray(format="bgr24")
    
# #     # Remove background using SelfiSegmentation
# #     imgOut = segmentor.removeBG(image, imgList[indexImg])

# #     # Detect hands on the background-removed image
# #     hands, img = detector.findHands(imgOut, flipType=False)
    
# #     # Create a blank canvas for the keyboard
# #     keyboard_canvas = np.zeros_like(img)
# #     buttonList = []

# #     # Create buttons for the virtual keyboard based on the keys list
# #     for key in keys[0]:
# #         buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
# #     for key in keys[1]:
# #         buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
# #     for key in keys[2]:
# #         buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))

# #     # Draw the buttons on the keyboard canvas
# #     for button in buttonList:
# #         x, y = button.pos
# #         cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
# #         cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)

# #     # Handle input and gestures from detected hands
# #     if hands:
# #         for hand in hands:
# #             lmList = hand["lmList"]
# #             if lmList:
# #                 # Get the coordinates of the index finger tip (landmark 8)
# #                 x8, y8 = lmList[8][0], lmList[8][1]
# #                 for button in buttonList:
# #                     bx, by = button.pos
# #                     bw, bh = button.size
# #                     # Check if the index finger is over a button
# #                     if bx < x8 < bx + bw and by < y8 < by + bh:
# #                         # Highlight the button and update the text
# #                         cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
# #                         cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
# #                         # Update the output text in session_state
# #                         session_state["output_text"] += button.text

# #     # Corrected return: Create a video frame from the ndarray image
# #     return av.VideoFrame.from_ndarray(img, format="bgr24")






# # Shared state for output text
# if "output_text" not in st.session_state:
#     st.session_state["output_text"] = ""

# class Detection(NamedTuple):
#     label: str
#     score: float
#     box: np.ndarray


# @st.cache_resource  # Cache label colors
# def generate_label_colors():
#     return np.random.uniform(0, 255, size=(2, 3))  # Two classes: Left and Right Hand


# COLORS = generate_label_colors()

# # Initialize MediaPipe Hands
# mp_hands = mp.solutions.hands
# detector = mp_hands.Hands(static_image_mode=False, max_num_hands=2, min_detection_confidence=0.5)

# # Session-specific caching
# result_queue: "queue.Queue[List[Detection]]" = queue.Queue()

# # Hand detection callback
# def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
#     image = frame.to_ndarray(format="bgr24")
#     h, w = image.shape[:2]

#     # Process image with MediaPipe Hands
#     results = detector.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))

#     detections = []
#     if results.multi_hand_landmarks:
#         for hand_landmarks, hand_class in zip(results.multi_hand_landmarks, results.multi_handedness):
#             # Extract bounding box
#             x_min, y_min = 1, 1
#             x_max, y_max = 0, 0
#             for lm in hand_landmarks.landmark:
#                 x_min = min(x_min, lm.x)
#                 y_min = min(y_min, lm.y)
#                 x_max = max(x_max, lm.x)
#                 y_max = max(y_max, lm.y)

#             # Scale bbox to image size
#             box = np.array([x_min * w, y_min * h, x_max * w, y_max * h]).astype("int")

#             # Label and score
#             label = hand_class.classification[0].label
#             score = hand_class.classification[0].score

#             detections.append(Detection(label=label, score=score, box=box))

#             # Draw bounding box and label
#             color = COLORS[0 if label == "Left" else 1]
#             cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), color, 2)
#             caption = f"{label}: {round(score * 100, 2)}%"
#             cv2.putText(
#                 image,
#                 caption,
#                 (box[0], box[1] - 15 if box[1] - 15 > 15 else box[1] + 15),
#                 cv2.FONT_HERSHEY_SIMPLEX,
#                 0.5,
#                 color,
#                 2,
#             )

#     # Put results in the queue
#     result_queue.put(detections)

#     return av.VideoFrame.from_ndarray(image, format="bgr24")



# webrtc_ctx = webrtc_streamer(
#     key="keyboard-demo",
#     mode=WebRtcMode.SENDRECV,
#     rtc_configuration={
#         "iceServers": get_ice_servers(),
#         "iceTransportPolicy": "relay",
#     },
#     video_frame_callback=video_frame_callback,
#     media_stream_constraints={"video": True, "audio": False},
#     async_processing=True,
# )


# st.markdown("### Instructions")
# st.write(
#     """
#     1. Turn on your webcam using the checkbox above.
#     2. Use hand gestures to interact with the virtual keyboard.
#     """
# ) 

#)

import logging
import cv2
import numpy as np
import mediapipe as mp
import streamlit as st
from streamlit_webrtc import webrtc_streamer
import av
import queue
from typing import List

# Logging setup
logger = logging.getLogger(__name__)

# Streamlit setup
st.title("AI Squat Detection using WebRTC")
st.info("Use your webcam for real-time squat detection.")

# Initialize MediaPipe components
mp_pose = mp.solutions.pose
mp_drawing = mp.solutions.drawing_utils

# Angle calculation function
def calculate_angle(a, b, c):
    a = np.array(a)
    b = np.array(b)
    c = np.array(c)
    radians = np.arctan2(c[1]-b[1], c[0]-b[0]) - np.arctan2(a[1]-b[1], a[0]-b[0])
    angle = np.abs(radians * 180.0 / np.pi)
    if angle > 180.0:
        angle = 360 - angle
    return angle


# Detection Queue
result_queue: queue.Queue[List[Detection]] = queue.Queue()

def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
    image = frame.to_ndarray(format="bgr24")
    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

    with mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5) as pose:
        results = pose.process(image_rgb)
        landmarks = results.pose_landmarks.landmark if results.pose_landmarks else []

        # Corrected detection logic
        detections = [
            Detection(
                class_id=0,  # Assuming a generic class_id for pose detections
                label="Pose",
                score=1.0,  # Full confidence as pose landmarks were detected
                box=np.array([0, 0, image.shape[1], image.shape[0]])  # Full image as bounding box
            )
        ] if landmarks else []

        if landmarks:
            hip = [landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].x, 
                   landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].y]
            knee = [landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].x, 
                    landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].y]
            ankle = [landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].x, 
                     landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].y]
            shoulder = [landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].x, 
                        landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].y]
            foot = [landmarks[mp_pose.PoseLandmark.LEFT_FOOT_INDEX.value].x, 
                    landmarks[mp_pose.PoseLandmark.LEFT_FOOT_INDEX.value].y]

            # Calculate angles
            knee_angle = calculate_angle(hip, knee, ankle)
            hip_angle = calculate_angle(shoulder, hip, [hip[0], 0])
            ankle_angle = calculate_angle(foot, ankle, knee)
            
            # Display key angles
            cv2.putText(image, f"Knee: {int(knee_angle)}", (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
            cv2.putText(image, f"Hip: {int(hip_angle)}", (10, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
            cv2.putText(image, f"Ankle: {int(ankle_angle)}", (10, 150), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
            
            # Squat logic
            if 80 < knee_angle < 110 and 29 < hip_angle < 40:
                cv2.putText(image, "Squat Detected!", (300, 100), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 255, 0), 3)
            else:
                if hip_angle < 29:
                    cv2.putText(image, "Lean Forward!", (300, 200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3)
                elif hip_angle > 45:
                    cv2.putText(image, "Lean Backward!", (300, 200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3)
                if knee_angle < 80:
                    cv2.putText(image, "Squat Too Deep!", (300, 250), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3)
                elif knee_angle > 110:
                    cv2.putText(image, "Lower Your Hips!", (300, 300), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3)

        mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS,
                                  mp_drawing.DrawingSpec(color=(255, 175, 0), thickness=2, circle_radius=2),
                                  mp_drawing.DrawingSpec(color=(0, 255, 200), thickness=2, circle_radius=2))

    result_queue.put(detections)
    return av.VideoFrame.from_ndarray(image, format="bgr24")



# WebRTC streamer configuration
webrtc_streamer(
    key="squat-detection",
    video_frame_callback=video_frame_callback,
    media_stream_constraints={"video": True, "audio": False},
    async_processing=True
)































# import logging
# import cv2
# import numpy as np
# import streamlit as st
# from streamlit_webrtc import WebRtcMode, webrtc_streamer
# from cvzone.HandTrackingModule import HandDetector
# from cvzone.SelfiSegmentationModule import SelfiSegmentation
# import os
# import time
# import av
# import queue
# from typing import List, NamedTuple
# from sample_utils.turn import get_ice_servers

# logger = logging.getLogger(__name__)

# # Streamlit settings
# st.set_page_config(page_title="Virtual Keyboard", layout="wide")
# st.title("Interactive Virtual Keyboard")
# st.subheader('''Turn on the webcam and use hand gestures to interact with the virtual keyboard.
# Use 'a' and 'd' from the keyboard to change the background.''')

# # Initialize modules
# detector = HandDetector(maxHands=1, detectionCon=0.8)
# segmentor = SelfiSegmentation()

# # Define virtual keyboard layout
# keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
#         ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
#         ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]

# class Button:
#     def __init__(self, pos, text, size=[100, 100]):
#         self.pos = pos
#         self.size = size
#         self.text = text

# class Detection(NamedTuple):
#     label: str
#     score: float
#     box: np.ndarray

# # result_queue: "queue.Queue[List[Detection]]" = queue.Queue()

# listImg = os.listdir('model/street') if os.path.exists('model/street') else []
# if not listImg:
#     st.error("Error: 'street' directory is missing or empty. Please add background images.")
#     st.stop()
# else:
#     imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg if cv2.imread(f'model/street/{imgPath}') is not None]

# indexImg = 0
# prev_key_time = [time.time()] * 2
# output_text = ""

# if "output_text" not in st.session_state:
#     st.session_state["output_text"] = ""


# # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
# #     img = frame.to_ndarray(format="bgr24")
# #     hands, img = detector.findHands(img, flipType=False)

# #     # Render hand detection results

# #     if hands:
# #         hand = hands[0]
# #         bbox = hand["bbox"]
# #         cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[0]+bbox[2], bbox[1]+bbox[3]), (255, 0, 0), 2)

# #         cv2.putText(img, 'OpenCV', (50,50), font, 
# #                    fontScale, color, thickness, cv2.LINE_AA)
# #         cv2.putText(img, 'OpenCV', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 1, cv2.LINE_AA)

# #         result_queue.put(hands)
        
# #     return av.VideoFrame.from_ndarray(img, format="bgr24")


# result_queue: "queue.Queue[List[Detection]]" = queue.Queue()


# def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
#     image = frame.to_ndarray(format="bgr24")

#     # Run inference
#     blob = cv2.dnn.blobFromImage(
#         cv2.resize(image, (300, 300)), 0.007843, (300, 300), 127.5
#     )
#     net.setInput(blob)
#     output = net.forward()

#     h, w = image.shape[:2]

#     # Convert the output array into a structured form.
#     output = output.squeeze()  # (1, 1, N, 7) -> (N, 7)
#     output = output[output[:, 2] >= score_threshold]
#     detections = [
#         Detection(
#             class_id=int(detection[1]),
#             label=CLASSES[int(detection[1])],
#             score=float(detection[2]),
#             box=(detection[3:7] * np.array([w, h, w, h])),
#         )
#         for detection in output
#     ]

#     # Render bounding boxes and captions
#     for detection in detections:
#         caption = f"{detection.label}: {round(detection.score * 100, 2)}%"
#         color = COLORS[detection.class_id]
#         xmin, ymin, xmax, ymax = detection.box.astype("int")

#         cv2.rectangle(image, (xmin, ymin), (xmax, ymax), color, 2)
#         cv2.putText(
#             image,
#             caption,
#             (xmin, ymin - 15 if ymin - 15 > 15 else ymin + 15),
#             cv2.FONT_HERSHEY_SIMPLEX,
#             0.5,
#             color,
#             2,
#         )

#     result_queue.put(detections)

#     return av.VideoFrame.from_ndarray(image, format="bgr24")


# # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
# #     global indexImg, output_text

# #     img = frame.to_ndarray(format="bgr24")
# #     imgOut = segmentor.removeBG(img, imgList[indexImg])
# #     hands, imgOut = detector.findHands(imgOut, flipType=False)

# #     buttonList = [Button([30 + col * 105, 30 + row * 120], key) for row, line in enumerate(keys) for col, key in enumerate(line)]

# #     detections = []
# #     if hands:
# #         for i, hand in enumerate(hands):
# #             lmList = hand['lmList']
# #             bbox = hand['bbox']
# #             label = "Hand"
# #             score = hand['score']
# #             box = np.array([bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]])
# #             detections.append(Detection(label=label, score=score, box=box))

# #             if lmList:
# #                 x4, y4 = lmList[4][0], lmList[4][1]
# #                 x8, y8 = lmList[8][0], lmList[8][1]
# #                 distance = np.sqrt((x8 - x4) ** 2 + (y8 - y4) ** 2)
# #                 click_threshold = 10

# #                 for button in buttonList:
# #                     x, y = button.pos
# #                     w, h = button.size
# #                     if x < x8 < x + w and y < y8 < y + h:
# #                         cv2.rectangle(imgOut, button.pos, (x + w, y + h), (0, 255, 160), -1)
# #                         cv2.putText(imgOut, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)

# #                         if (distance / np.sqrt(bbox[2] ** 2 + bbox[3] ** 2)) * 100 < click_threshold:
# #                             if time.time() - prev_key_time[i] > 2:
# #                                 prev_key_time[i] = time.time()
# #                                 if button.text != 'BS' and button.text != 'SPACE':
# #                                     output_text += button.text
# #                                 elif button.text == 'BS':
# #                                     output_text = output_text[:-1]
# #                                 else:
# #                                     output_text += ' '

# #     result_queue.put(detections)
# #     st.session_state["output_text"] = output_text
# #     return av.VideoFrame.from_ndarray(imgOut, format="bgr24")

    

# webrtc_streamer(
#     key="virtual-keyboard",
#     mode=WebRtcMode.SENDRECV,
#     rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"},
#     media_stream_constraints={"video": True, "audio": False},
#     video_frame_callback=video_frame_callback,
#     async_processing=True,
# )

# st.subheader("Output Text")
# st.text_area("Live Input:", value=st.session_state["output_text"], height=200)