| import logging | |
| import queue | |
| from pathlib import Path | |
| from typing import List, NamedTuple | |
| import mediapipe as mp | |
| import av | |
| import cv2 | |
| import numpy as np | |
| import streamlit as st | |
| from streamlit_webrtc import WebRtcMode, webrtc_streamer | |
| from sample_utils.download import download_file | |
| from sample_utils.turn import get_ice_servers | |
| # Logging setup | |
| logger = logging.getLogger(__name__) | |
| # Streamlit setup | |
| st.title("AI Squat Detection using WebRTC") | |
| st.info("Use your webcam for real-time squat detection.") | |
| # Initialize MediaPipe components | |
| mp_pose = mp.solutions.pose | |
| mp_drawing = mp.solutions.drawing_utils | |
| class Detection(NamedTuple): | |
| class_id: int | |
| label: str | |
| score: float | |
| box: np.ndarray | |
| # Angle calculation function | |
| def calculate_angle(a, b, c): | |
| a = np.array(a) | |
| b = np.array(b) | |
| c = np.array(c) | |
| radians = np.arctan2(c[1]-b[1], c[0]-b[0]) - np.arctan2(a[1]-b[1], a[0]-b[0]) | |
| angle = np.abs(radians * 180.0 / np.pi) | |
| if angle > 180.0: | |
| angle = 360 - angle | |
| return angle | |
| counterL=0#Counter checks for number of curls | |
| correct=0 | |
| incorrect=0 | |
| stage=None#it checks if we our hand is UP or DOWN | |
| # Detection Queue | |
| result_queue: "queue.Queue[List[Detection]]" = queue.Queue() | |
| def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
| image = frame.to_ndarray(format="bgr24") | |
| h, w = image.shape[:2] | |
| image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) | |
| with mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5) as pose: | |
| results = pose.process(image_rgb) | |
| landmarks = results.pose_landmarks.landmark if results.pose_landmarks else [] | |
| # Corrected detection logic | |
| detections = [ | |
| Detection( | |
| class_id=0, # Assuming a generic class_id for pose detections | |
| label="Pose", | |
| score=0.7, # Full confidence as pose landmarks were detected | |
| box=np.array([0, 0, image.shape[1], image.shape[0]]) # Full image as bounding box | |
| ) | |
| ] if landmarks else [] | |
| if landmarks: | |
| hipL = [landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].x, | |
| landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].y] | |
| kneeL = [landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].x, | |
| landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].y] | |
| ankleL = [landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].x, | |
| landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].y] | |
| shoulderL = [landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].x, | |
| landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].y] | |
| footIndexL = [landmarks[mp_pose.PoseLandmark.LEFT_FOOT_INDEX.value].x, | |
| landmarks[mp_pose.PoseLandmark.LEFT_FOOT_INDEX.value].y] | |
| # Calculate angles | |
| angleKneeL = calculate_angle(hipL, kneeL, ankleL) | |
| angleHipL = calculate_angle(shoulderL, hipL, [hipL[0], 0]) | |
| angleAnkleL = calculate_angle(footIndexL, ankleL, kneeL) | |
| #Visualize of left leg | |
| cv2.putText(image, str(angleHipL),tuple(np.multiply(angleHipL, [640, 480]).astype(int)),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2, cv2.LINE_AA) | |
| # # Squat logic | |
| # if 80 < angleKneeL < 110 and 29 < angleHipL < 40: | |
| # cv2.putText(image, "Squat Detected!", (300, 100), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 255, 0), 3) | |
| # else: | |
| # if angleHipL < 29: | |
| # cv2.putText(image, "Lean Forward!", (300, 200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3) | |
| # elif angleHipL > 45: | |
| # cv2.putText(image, "Lean Backward!", (300, 200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3) | |
| # if angleKneeL < 80: | |
| # cv2.putText(image, "Squat Too Deep!", (300, 250), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3) | |
| # elif angleKneeL > 110: | |
| # cv2.putText(image, "Lower Your Hips!", (300, 300), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3) | |
| # 1. Bend Forward Warning | |
| if 10 < angleHipL < 18: | |
| cv2.rectangle(image, (310, 180), (450, 220), (0, 0, 0), -1) | |
| cv2.putText(image,f"Bend Forward",(320,200),cv2.FONT_HERSHEY_SIMPLEX,1,(150,120,255),1,cv2.LINE_AA) | |
| # 2. Lean Backward Warning | |
| if angleHipL > 45: | |
| cv2.rectangle(image, (310, 180), (450, 220), (0, 0, 0), -1) | |
| cv2.putText(image,f"Bend Backward",(320,200),cv2.FONT_HERSHEY_SIMPLEX,1,(80,120,255),1,cv2.LINE_AA) | |
| # # stage 2 | |
| # # Incorrect movements | |
| # 3. Knees not low enough | |
| if 110 < angleKneeL < 130: | |
| cv2.rectangle(image, (220, 40), (450, 80), (0, 0, 0), -1) | |
| cv2.putText(image,f"Lower Your Hips",(230,60),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,255),1,cv2.LINE_AA) | |
| # # 3. Knees not low enough and not completed the squat | |
| # if angleKneeL>130 and stage=='mid': | |
| # cv2.rectangle(image, (220, 40), (450, 80), (0, 0, 0), -1) | |
| # cv2.putText(image,f"Lower Your Hips",(230,60),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,255),1,cv2.LINE_AA) | |
| # incorrect+=1 | |
| # stage='up' | |
| # # 4. Squat too deep | |
| # if angleKneeL < 80 and stage=='mid': | |
| # cv2.rectangle(image, (220, 40), (450, 80), (0, 0, 0), -1) | |
| # cv2.putText(image,f"Squat too deep",(230,60),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,255),1,cv2.LINE_AA) | |
| # incorrect +=1 | |
| # stage='up' | |
| # # stage 4 | |
| # if (80 < angleKneeL < 110) and stage=='mid': | |
| # if (18 < angleHipL < 40): # Valid "down" position | |
| # correct+=1 | |
| # stage='up' | |
| # cv2.putText(image,f"Correct:{correct}", | |
| # (400,120),cv2.FONT_HERSHEY_SIMPLEX,1,(0,0,0),1,cv2.LINE_AA) | |
| # cv2.putText(image,f"Incorrect:{incorrect}", | |
| # (400,150),cv2.FONT_HERSHEY_SIMPLEX,1,(0,0,0),1,cv2.LINE_AA) | |
| #Render Counter to our camera screen | |
| #Setup Status box | |
| cv2.rectangle(image,(0,0),(500,80),(245,117,16),-1) | |
| #REP data | |
| cv2.putText(image,'Left',(10,12), | |
| cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,0),1,cv2.LINE_AA) | |
| cv2.putText(image,str(correct), | |
| (10,60),cv2.FONT_HERSHEY_SIMPLEX,2,(255,255,255),2,cv2.LINE_AA) | |
| #Stage data for left leg | |
| cv2.putText(image,'STAGE',(230,12), | |
| cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,0),1,cv2.LINE_AA) | |
| cv2.putText(image,stage, | |
| (230,60),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,255),1,cv2.LINE_AA) | |
| mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS,mp_drawing.DrawingSpec(color=(255, 175, 0), thickness=2, circle_radius=2),mp_drawing.DrawingSpec(color=(0, 255, 200), thickness=2, circle_radius=2)) | |
| result_queue.put(detections) | |
| return av.VideoFrame.from_ndarray(image, format="bgr24") | |
| # WebRTC streamer configuration | |
| webrtc_streamer( | |
| key="squat-detection", | |
| mode=WebRtcMode.SENDRECV, | |
| rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"}, | |
| media_stream_constraints={"video": True, "audio": False}, | |
| video_frame_callback=video_frame_callback, | |
| async_processing=True, | |
| ) | |
| # import logging | |
| # import cv2 | |
| # import numpy as np | |
| # import streamlit as st | |
| # from streamlit_webrtc import WebRtcMode, webrtc_streamer | |
| # from cvzone.HandTrackingModule import HandDetector | |
| # from cvzone.SelfiSegmentationModule import SelfiSegmentation | |
| # import os | |
| # import time | |
| # import av | |
| # import queue | |
| # from typing import List, NamedTuple | |
| # from sample_utils.turn import get_ice_servers | |
| # logger = logging.getLogger(__name__) | |
| # # Streamlit settings | |
| # st.set_page_config(page_title="Virtual Keyboard", layout="wide") | |
| # st.title("Interactive Virtual Keyboard") | |
| # st.subheader('''Turn on the webcam and use hand gestures to interact with the virtual keyboard. | |
| # Use 'a' and 'd' from the keyboard to change the background.''') | |
| # # Initialize modules | |
| # detector = HandDetector(maxHands=1, detectionCon=0.8) | |
| # segmentor = SelfiSegmentation() | |
| # # Define virtual keyboard layout | |
| # keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"], | |
| # ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"], | |
| # ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]] | |
| # class Button: | |
| # def __init__(self, pos, text, size=[100, 100]): | |
| # self.pos = pos | |
| # self.size = size | |
| # self.text = text | |
| # class Detection(NamedTuple): | |
| # label: str | |
| # score: float | |
| # box: np.ndarray | |
| # # result_queue: "queue.Queue[List[Detection]]" = queue.Queue() | |
| # listImg = os.listdir('model/street') if os.path.exists('model/street') else [] | |
| # if not listImg: | |
| # st.error("Error: 'street' directory is missing or empty. Please add background images.") | |
| # st.stop() | |
| # else: | |
| # imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg if cv2.imread(f'model/street/{imgPath}') is not None] | |
| # indexImg = 0 | |
| # prev_key_time = [time.time()] * 2 | |
| # output_text = "" | |
| # if "output_text" not in st.session_state: | |
| # st.session_state["output_text"] = "" | |
| # # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
| # # img = frame.to_ndarray(format="bgr24") | |
| # # hands, img = detector.findHands(img, flipType=False) | |
| # # # Render hand detection results | |
| # # if hands: | |
| # # hand = hands[0] | |
| # # bbox = hand["bbox"] | |
| # # cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[0]+bbox[2], bbox[1]+bbox[3]), (255, 0, 0), 2) | |
| # # cv2.putText(img, 'OpenCV', (50,50), font, | |
| # # fontScale, color, thickness, cv2.LINE_AA) | |
| # # cv2.putText(img, 'OpenCV', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 1, cv2.LINE_AA) | |
| # # result_queue.put(hands) | |
| # # return av.VideoFrame.from_ndarray(img, format="bgr24") | |
| # result_queue: "queue.Queue[List[Detection]]" = queue.Queue() | |
| # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
| # image = frame.to_ndarray(format="bgr24") | |
| # # Run inference | |
| # blob = cv2.dnn.blobFromImage( | |
| # cv2.resize(image, (300, 300)), 0.007843, (300, 300), 127.5 | |
| # ) | |
| # net.setInput(blob) | |
| # output = net.forward() | |
| # h, w = image.shape[:2] | |
| # # Convert the output array into a structured form. | |
| # output = output.squeeze() # (1, 1, N, 7) -> (N, 7) | |
| # output = output[output[:, 2] >= score_threshold] | |
| # detections = [ | |
| # Detection( | |
| # class_id=int(detection[1]), | |
| # label=CLASSES[int(detection[1])], | |
| # score=float(detection[2]), | |
| # box=(detection[3:7] * np.array([w, h, w, h])), | |
| # ) | |
| # for detection in output | |
| # ] | |
| # # Render bounding boxes and captions | |
| # for detection in detections: | |
| # caption = f"{detection.label}: {round(detection.score * 100, 2)}%" | |
| # color = COLORS[detection.class_id] | |
| # xmin, ymin, xmax, ymax = detection.box.astype("int") | |
| # cv2.rectangle(image, (xmin, ymin), (xmax, ymax), color, 2) | |
| # cv2.putText( | |
| # image, | |
| # caption, | |
| # (xmin, ymin - 15 if ymin - 15 > 15 else ymin + 15), | |
| # cv2.FONT_HERSHEY_SIMPLEX, | |
| # 0.5, | |
| # color, | |
| # 2, | |
| # ) | |
| # result_queue.put(detections) | |
| # return av.VideoFrame.from_ndarray(image, format="bgr24") | |
| # # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
| # # global indexImg, output_text | |
| # # img = frame.to_ndarray(format="bgr24") | |
| # # imgOut = segmentor.removeBG(img, imgList[indexImg]) | |
| # # hands, imgOut = detector.findHands(imgOut, flipType=False) | |
| # # buttonList = [Button([30 + col * 105, 30 + row * 120], key) for row, line in enumerate(keys) for col, key in enumerate(line)] | |
| # # detections = [] | |
| # # if hands: | |
| # # for i, hand in enumerate(hands): | |
| # # lmList = hand['lmList'] | |
| # # bbox = hand['bbox'] | |
| # # label = "Hand" | |
| # # score = hand['score'] | |
| # # box = np.array([bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]]) | |
| # # detections.append(Detection(label=label, score=score, box=box)) | |
| # # if lmList: | |
| # # x4, y4 = lmList[4][0], lmList[4][1] | |
| # # x8, y8 = lmList[8][0], lmList[8][1] | |
| # # distance = np.sqrt((x8 - x4) ** 2 + (y8 - y4) ** 2) | |
| # # click_threshold = 10 | |
| # # for button in buttonList: | |
| # # x, y = button.pos | |
| # # w, h = button.size | |
| # # if x < x8 < x + w and y < y8 < y + h: | |
| # # cv2.rectangle(imgOut, button.pos, (x + w, y + h), (0, 255, 160), -1) | |
| # # cv2.putText(imgOut, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3) | |
| # # if (distance / np.sqrt(bbox[2] ** 2 + bbox[3] ** 2)) * 100 < click_threshold: | |
| # # if time.time() - prev_key_time[i] > 2: | |
| # # prev_key_time[i] = time.time() | |
| # # if button.text != 'BS' and button.text != 'SPACE': | |
| # # output_text += button.text | |
| # # elif button.text == 'BS': | |
| # # output_text = output_text[:-1] | |
| # # else: | |
| # # output_text += ' ' | |
| # # result_queue.put(detections) | |
| # # st.session_state["output_text"] = output_text | |
| # # return av.VideoFrame.from_ndarray(imgOut, format="bgr24") | |
| # webrtc_streamer( | |
| # key="virtual-keyboard", | |
| # mode=WebRtcMode.SENDRECV, | |
| # rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"}, | |
| # media_stream_constraints={"video": True, "audio": False}, | |
| # video_frame_callback=video_frame_callback, | |
| # async_processing=True, | |
| # ) | |
| # st.subheader("Output Text") | |
| # st.text_area("Live Input:", value=st.session_state["output_text"], height=200) | |