# import logging | |
# import queue | |
# from pathlib import Path | |
# from typing import List, NamedTuple | |
# import mediapipe as mp | |
# import av | |
# import cv2 | |
# import numpy as np | |
# import streamlit as st | |
# from streamlit_webrtc import WebRtcMode, webrtc_streamer | |
# from sample_utils.turn import get_ice_servers | |
# from cvzone.HandTrackingModule import HandDetector | |
# from cvzone.SelfiSegmentationModule import SelfiSegmentation | |
# import time | |
# import os | |
# logger = logging.getLogger(__name__) | |
# st.title("Interactive Virtual Keyboard with Twilio Integration") | |
# st.info("Use your webcam to interact with the virtual keyboard via hand gestures.") | |
# class Button: | |
# def __init__(self, pos, text, size=[100, 100]): | |
# self.pos = pos | |
# self.size = size | |
# self.text = text | |
# # Initialize components | |
# detector = HandDetector(maxHands=1, detectionCon=0.8) | |
# # segmentor = SelfiSegmentation() | |
# # keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"], | |
# # ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"], | |
# # ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]] | |
# # listImg = os.listdir('model/street') | |
# # imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg] | |
# # indexImg = 0 | |
# # # Function to process the video frame from the webcam | |
# # def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state): | |
# # # Convert the frame to a numpy array (BGR format) | |
# # image = frame.to_ndarray(format="bgr24") | |
# # # Remove background using SelfiSegmentation | |
# # imgOut = segmentor.removeBG(image, imgList[indexImg]) | |
# # # Detect hands on the background-removed image | |
# # hands, img = detector.findHands(imgOut, flipType=False) | |
# # # Create a blank canvas for the keyboard | |
# # keyboard_canvas = np.zeros_like(img) | |
# # buttonList = [] | |
# # # Create buttons for the virtual keyboard based on the keys list | |
# # for key in keys[0]: | |
# # buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key)) | |
# # for key in keys[1]: | |
# # buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key)) | |
# # for key in keys[2]: | |
# # buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key)) | |
# # # Draw the buttons on the keyboard canvas | |
# # for button in buttonList: | |
# # x, y = button.pos | |
# # cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1) | |
# # cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3) | |
# # # Handle input and gestures from detected hands | |
# # if hands: | |
# # for hand in hands: | |
# # lmList = hand["lmList"] | |
# # if lmList: | |
# # # Get the coordinates of the index finger tip (landmark 8) | |
# # x8, y8 = lmList[8][0], lmList[8][1] | |
# # for button in buttonList: | |
# # bx, by = button.pos | |
# # bw, bh = button.size | |
# # # Check if the index finger is over a button | |
# # if bx < x8 < bx + bw and by < y8 < by + bh: | |
# # # Highlight the button and update the text | |
# # cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1) | |
# # cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3) | |
# # # Update the output text in session_state | |
# # session_state["output_text"] += button.text | |
# # # Corrected return: Create a video frame from the ndarray image | |
# # return av.VideoFrame.from_ndarray(img, format="bgr24") | |
# # Shared state for output text | |
# if "output_text" not in st.session_state: | |
# st.session_state["output_text"] = "" | |
# class Detection(NamedTuple): | |
# label: str | |
# score: float | |
# box: np.ndarray | |
# @st.cache_resource # Cache label colors | |
# def generate_label_colors(): | |
# return np.random.uniform(0, 255, size=(2, 3)) # Two classes: Left and Right Hand | |
# COLORS = generate_label_colors() | |
# # Initialize MediaPipe Hands | |
# mp_hands = mp.solutions.hands | |
# detector = mp_hands.Hands(static_image_mode=False, max_num_hands=2, min_detection_confidence=0.5) | |
# # Session-specific caching | |
# result_queue: "queue.Queue[List[Detection]]" = queue.Queue() | |
# # Hand detection callback | |
# def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
# image = frame.to_ndarray(format="bgr24") | |
# h, w = image.shape[:2] | |
# # Process image with MediaPipe Hands | |
# results = detector.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) | |
# detections = [] | |
# if results.multi_hand_landmarks: | |
# for hand_landmarks, hand_class in zip(results.multi_hand_landmarks, results.multi_handedness): | |
# # Extract bounding box | |
# x_min, y_min = 1, 1 | |
# x_max, y_max = 0, 0 | |
# for lm in hand_landmarks.landmark: | |
# x_min = min(x_min, lm.x) | |
# y_min = min(y_min, lm.y) | |
# x_max = max(x_max, lm.x) | |
# y_max = max(y_max, lm.y) | |
# # Scale bbox to image size | |
# box = np.array([x_min * w, y_min * h, x_max * w, y_max * h]).astype("int") | |
# # Label and score | |
# label = hand_class.classification[0].label | |
# score = hand_class.classification[0].score | |
# detections.append(Detection(label=label, score=score, box=box)) | |
# # Draw bounding box and label | |
# color = COLORS[0 if label == "Left" else 1] | |
# cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), color, 2) | |
# caption = f"{label}: {round(score * 100, 2)}%" | |
# cv2.putText( | |
# image, | |
# caption, | |
# (box[0], box[1] - 15 if box[1] - 15 > 15 else box[1] + 15), | |
# cv2.FONT_HERSHEY_SIMPLEX, | |
# 0.5, | |
# color, | |
# 2, | |
# ) | |
# # Put results in the queue | |
# result_queue.put(detections) | |
# return av.VideoFrame.from_ndarray(image, format="bgr24") | |
# webrtc_ctx = webrtc_streamer( | |
# key="keyboard-demo", | |
# mode=WebRtcMode.SENDRECV, | |
# rtc_configuration={ | |
# "iceServers": get_ice_servers(), | |
# "iceTransportPolicy": "relay", | |
# }, | |
# video_frame_callback=video_frame_callback, | |
# media_stream_constraints={"video": True, "audio": False}, | |
# async_processing=True, | |
# ) | |
# st.markdown("### Instructions") | |
# st.write( | |
# """ | |
# 1. Turn on your webcam using the checkbox above. | |
# 2. Use hand gestures to interact with the virtual keyboard. | |
# """ | |
# ) | |
#) | |
import logging | |
import queue | |
from pathlib import Path | |
from typing import List, NamedTuple | |
import mediapipe as mp | |
import av | |
import cv2 | |
import numpy as np | |
import streamlit as st | |
from streamlit_webrtc import WebRtcMode, webrtc_streamer | |
from sample_utils.download import download_file | |
from sample_utils.turn import get_ice_servers | |
# Logging setup | |
logger = logging.getLogger(__name__) | |
# Streamlit setup | |
st.title("AI Squat Detection using WebRTC") | |
st.info("Use your webcam for real-time squat detection.") | |
# Initialize MediaPipe components | |
mp_pose = mp.solutions.pose | |
mp_drawing = mp.solutions.drawing_utils | |
class Detection(NamedTuple): | |
class_id: int | |
label: str | |
score: float | |
box: np.ndarray | |
# Angle calculation function | |
def calculate_angle(a, b, c): | |
a = np.array(a) | |
b = np.array(b) | |
c = np.array(c) | |
radians = np.arctan2(c[1] - b[1], c[0] - b[0]) - np.arctan2(a[1] - b[1], a[0] - b[0]) | |
angle = np.abs(radians * 180.0 / np.pi) | |
if angle > 180.0: | |
angle = 360 - angle | |
return angle | |
counterL = 0 # Counter checks for number of curls | |
correct = 0 | |
incorrect = 0 | |
stage = None # it checks if we our hand is UP or DOWN | |
# Detection Queue | |
result_queue: "queue.Queue[List[Detection]]" = queue.Queue() | |
def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
image = frame.to_ndarray(format="bgr24") | |
h, w = image.shape[:2] | |
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) | |
with mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5) as pose: | |
results = pose.process(image_rgb) | |
# If landmarks are detected, proceed | |
if results.pose_landmarks: | |
landmarks = results.pose_landmarks.landmark | |
else: | |
landmarks = [] | |
cv2.putText(image, "No Pose Detected", (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv2.LINE_AA) | |
# Corrected detection logic | |
detections = [ | |
Detection( | |
class_id=0, # Assuming a generic class_id for pose detections | |
label="Pose", | |
score=0.7, # Full confidence as pose landmarks were detected | |
box=np.array([0, 0, image.shape[1], image.shape[0]]) # Full image as bounding box | |
) | |
] if landmarks else [] | |
if landmarks: | |
hipL = [landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].x, | |
landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].y] | |
kneeL = [landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].x, | |
landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].y] | |
ankleL = [landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].x, | |
landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].y] | |
shoulderL = [landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].x, | |
landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].y] | |
footIndexL = [landmarks[mp_pose.PoseLandmark.LEFT_FOOT_INDEX.value].x, | |
landmarks[mp_pose.PoseLandmark.LEFT_FOOT_INDEX.value].y] | |
# Calculate angles | |
angleKneeL = calculate_angle(hipL, kneeL, ankleL) | |
angleHipL = calculate_angle(shoulderL, hipL, [hipL[0], 0]) | |
angleAnkleL = calculate_angle(footIndexL, ankleL, kneeL) | |
# Visualize of left leg | |
cv2.putText(image, str(angleHipL), tuple(np.multiply(angleHipL, [640, 480]).astype(int)), | |
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2, cv2.LINE_AA) | |
#Squat Logic | |
# 1. Bend Forward Warning | |
if 10 < angleHipL < 18: | |
print(f"AngleHipL when Bend forward warning:{angleHipL}") | |
cv2.rectangle(image, (310, 180), (450, 220), (0, 0, 0), -1) | |
cv2.putText(image, f"Bend Forward", (320, 200), cv2.FONT_HERSHEY_SIMPLEX, 1, (150, 120, 255), 1, cv2.LINE_AA) | |
# 2. Lean Backward Warning | |
if angleHipL > 45: | |
print(f"AngleHipL when Bend backward warning:{angleHipL}") | |
cv2.rectangle(image, (310, 180), (450, 220), (0, 0, 0), -1) | |
cv2.putText(image, f"Bend Backward", (320, 200), cv2.FONT_HERSHEY_SIMPLEX, 1, (80, 120, 255), 1, cv2.LINE_AA) | |
# Incorrect movements | |
# 3. Knees not low enough | |
if 110 < angleKneeL < 130: | |
print(f"AngleKneeL when Lower Your Hips warning:{angleKneeL}") | |
cv2.rectangle(image, (220, 40), (450, 80), (0, 0, 0), -1) | |
cv2.putText(image, f"Lower Your Hips", (230, 60), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv2.LINE_AA) | |
# 3. Knees not low enough and not completed the squat | |
if angleKneeL > 130 and stage == 'mid': | |
print(f"AngleKneeL when Knees not low enough and not completed the squat :{angleKneeL}") | |
cv2.rectangle(image, (220, 40), (450, 80), (0, 0, 0), -1) | |
cv2.putText(image, f"Lower Your Hips", (230, 60), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv2.LINE_AA) | |
print(f"Incorrect counter Knees not low enough and not completed the squat :{incorrect}") | |
incorrect += 1 | |
stage = 'up' | |
# 4. Squat too deep | |
if angleKneeL < 80 and stage == 'mid': | |
print(f"AngleKneeL when Squat too deep warning:{angleKneeL}") | |
cv2.rectangle(image, (220, 40), (450, 80), (0, 0, 0), -1) | |
cv2.putText(image, f"Squat too deep", (230, 60), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv2.LINE_AA) | |
print(f"Incorrect counter when Squat too deep warning:{incorrect}") | |
incorrect += 1 | |
stage = 'up' | |
# stage 4 | |
if (80 < angleKneeL < 110) and stage == 'mid': | |
if (18 < angleHipL < 40): # Valid "down" position | |
print(f"AngleKneeL when valid down position:{angleKneeL}") | |
print(f"AngleHipL when valid down position:{angleHipL}") | |
print(f"Correct counter when valid down position:{correct}") | |
correct += 1 | |
stage = 'up' | |
cv2.putText(image, f"Correct:{correct}", (400, 120), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 1, cv2.LINE_AA) | |
cv2.putText(image, f"Incorrect:{incorrect}", (400, 150), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 1, cv2.LINE_AA) | |
# Render Counter to our camera screen | |
# Setup Status box | |
cv2.rectangle(image, (0, 0), (500, 80), (245, 117, 16), -1) | |
# REP data | |
cv2.putText(image, 'Left', (10, 12), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA) | |
cv2.putText(image, str(correct), (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 2, cv2.LINE_AA) | |
# Stage data for left leg | |
cv2.putText(image, 'STAGE', (230, 12), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA) | |
cv2.putText(image, stage, (230, 60), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv2.LINE_AA) | |
mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS, | |
mp_drawing.DrawingSpec(color=(255, 175, 0), thickness=2, circle_radius=2), | |
mp_drawing.DrawingSpec(color=(0, 255, 200), thickness=2, circle_radius=2)) | |
result_queue.put(detections) | |
return av.VideoFrame.from_ndarray(image, format="bgr24") | |
# WebRTC streamer configuration | |
webrtc_streamer( | |
key="squat-detection", | |
mode=WebRtcMode.SENDRECV, | |
rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"}, | |
media_stream_constraints={"video": True, "audio": False}, | |
video_frame_callback=video_frame_callback, | |
async_processing=True, | |
) | |
# import logging | |
# import cv2 | |
# import numpy as np | |
# import streamlit as st | |
# from streamlit_webrtc import WebRtcMode, webrtc_streamer | |
# from cvzone.HandTrackingModule import HandDetector | |
# from cvzone.SelfiSegmentationModule import SelfiSegmentation | |
# import os | |
# import time | |
# import av | |
# import queue | |
# from typing import List, NamedTuple | |
# from sample_utils.turn import get_ice_servers | |
# logger = logging.getLogger(__name__) | |
# # Streamlit settings | |
# st.set_page_config(page_title="Virtual Keyboard", layout="wide") | |
# st.title("Interactive Virtual Keyboard") | |
# st.subheader('''Turn on the webcam and use hand gestures to interact with the virtual keyboard. | |
# Use 'a' and 'd' from the keyboard to change the background.''') | |
# # Initialize modules | |
# detector = HandDetector(maxHands=1, detectionCon=0.8) | |
# segmentor = SelfiSegmentation() | |
# # Define virtual keyboard layout | |
# keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"], | |
# ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"], | |
# ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]] | |
# class Button: | |
# def __init__(self, pos, text, size=[100, 100]): | |
# self.pos = pos | |
# self.size = size | |
# self.text = text | |
# class Detection(NamedTuple): | |
# label: str | |
# score: float | |
# box: np.ndarray | |
# # result_queue: "queue.Queue[List[Detection]]" = queue.Queue() | |
# listImg = os.listdir('model/street') if os.path.exists('model/street') else [] | |
# if not listImg: | |
# st.error("Error: 'street' directory is missing or empty. Please add background images.") | |
# st.stop() | |
# else: | |
# imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg if cv2.imread(f'model/street/{imgPath}') is not None] | |
# indexImg = 0 | |
# prev_key_time = [time.time()] * 2 | |
# output_text = "" | |
# if "output_text" not in st.session_state: | |
# st.session_state["output_text"] = "" | |
# # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
# # img = frame.to_ndarray(format="bgr24") | |
# # hands, img = detector.findHands(img, flipType=False) | |
# # # Render hand detection results | |
# # if hands: | |
# # hand = hands[0] | |
# # bbox = hand["bbox"] | |
# # cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[0]+bbox[2], bbox[1]+bbox[3]), (255, 0, 0), 2) | |
# # cv2.putText(img, 'OpenCV', (50,50), font, | |
# # fontScale, color, thickness, cv2.LINE_AA) | |
# # cv2.putText(img, 'OpenCV', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 1, cv2.LINE_AA) | |
# # result_queue.put(hands) | |
# # return av.VideoFrame.from_ndarray(img, format="bgr24") | |
# result_queue: "queue.Queue[List[Detection]]" = queue.Queue() | |
# def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
# image = frame.to_ndarray(format="bgr24") | |
# # Run inference | |
# blob = cv2.dnn.blobFromImage( | |
# cv2.resize(image, (300, 300)), 0.007843, (300, 300), 127.5 | |
# ) | |
# net.setInput(blob) | |
# output = net.forward() | |
# h, w = image.shape[:2] | |
# # Convert the output array into a structured form. | |
# output = output.squeeze() # (1, 1, N, 7) -> (N, 7) | |
# output = output[output[:, 2] >= score_threshold] | |
# detections = [ | |
# Detection( | |
# class_id=int(detection[1]), | |
# label=CLASSES[int(detection[1])], | |
# score=float(detection[2]), | |
# box=(detection[3:7] * np.array([w, h, w, h])), | |
# ) | |
# for detection in output | |
# ] | |
# # Render bounding boxes and captions | |
# for detection in detections: | |
# caption = f"{detection.label}: {round(detection.score * 100, 2)}%" | |
# color = COLORS[detection.class_id] | |
# xmin, ymin, xmax, ymax = detection.box.astype("int") | |
# cv2.rectangle(image, (xmin, ymin), (xmax, ymax), color, 2) | |
# cv2.putText( | |
# image, | |
# caption, | |
# (xmin, ymin - 15 if ymin - 15 > 15 else ymin + 15), | |
# cv2.FONT_HERSHEY_SIMPLEX, | |
# 0.5, | |
# color, | |
# 2, | |
# ) | |
# result_queue.put(detections) | |
# return av.VideoFrame.from_ndarray(image, format="bgr24") | |
# # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
# # global indexImg, output_text | |
# # img = frame.to_ndarray(format="bgr24") | |
# # imgOut = segmentor.removeBG(img, imgList[indexImg]) | |
# # hands, imgOut = detector.findHands(imgOut, flipType=False) | |
# # buttonList = [Button([30 + col * 105, 30 + row * 120], key) for row, line in enumerate(keys) for col, key in enumerate(line)] | |
# # detections = [] | |
# # if hands: | |
# # for i, hand in enumerate(hands): | |
# # lmList = hand['lmList'] | |
# # bbox = hand['bbox'] | |
# # label = "Hand" | |
# # score = hand['score'] | |
# # box = np.array([bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]]) | |
# # detections.append(Detection(label=label, score=score, box=box)) | |
# # if lmList: | |
# # x4, y4 = lmList[4][0], lmList[4][1] | |
# # x8, y8 = lmList[8][0], lmList[8][1] | |
# # distance = np.sqrt((x8 - x4) ** 2 + (y8 - y4) ** 2) | |
# # click_threshold = 10 | |
# # for button in buttonList: | |
# # x, y = button.pos | |
# # w, h = button.size | |
# # if x < x8 < x + w and y < y8 < y + h: | |
# # cv2.rectangle(imgOut, button.pos, (x + w, y + h), (0, 255, 160), -1) | |
# # cv2.putText(imgOut, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3) | |
# # if (distance / np.sqrt(bbox[2] ** 2 + bbox[3] ** 2)) * 100 < click_threshold: | |
# # if time.time() - prev_key_time[i] > 2: | |
# # prev_key_time[i] = time.time() | |
# # if button.text != 'BS' and button.text != 'SPACE': | |
# # output_text += button.text | |
# # elif button.text == 'BS': | |
# # output_text = output_text[:-1] | |
# # else: | |
# # output_text += ' ' | |
# # result_queue.put(detections) | |
# # st.session_state["output_text"] = output_text | |
# # return av.VideoFrame.from_ndarray(imgOut, format="bgr24") | |
# webrtc_streamer( | |
# key="virtual-keyboard", | |
# mode=WebRtcMode.SENDRECV, | |
# rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"}, | |
# media_stream_constraints={"video": True, "audio": False}, | |
# video_frame_callback=video_frame_callback, | |
# async_processing=True, | |
# ) | |
# st.subheader("Output Text") | |
# st.text_area("Live Input:", value=st.session_state["output_text"], height=200) | |