# import logging # import queue # from pathlib import Path # from typing import List, NamedTuple # import mediapipe as mp # import av # import cv2 # import numpy as np # import streamlit as st # from streamlit_webrtc import WebRtcMode, webrtc_streamer # from sample_utils.turn import get_ice_servers # from cvzone.HandTrackingModule import HandDetector # from cvzone.SelfiSegmentationModule import SelfiSegmentation # import time # import os # logger = logging.getLogger(__name__) # st.title("Interactive Virtual Keyboard with Twilio Integration") # st.info("Use your webcam to interact with the virtual keyboard via hand gestures.") # class Button: # def __init__(self, pos, text, size=[100, 100]): # self.pos = pos # self.size = size # self.text = text # # Initialize components # detector = HandDetector(maxHands=1, detectionCon=0.8) # # segmentor = SelfiSegmentation() # # keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"], # # ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"], # # ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]] # # listImg = os.listdir('model/street') # # imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg] # # indexImg = 0 # # # Function to process the video frame from the webcam # # def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state): # # # Convert the frame to a numpy array (BGR format) # # image = frame.to_ndarray(format="bgr24") # # # Remove background using SelfiSegmentation # # imgOut = segmentor.removeBG(image, imgList[indexImg]) # # # Detect hands on the background-removed image # # hands, img = detector.findHands(imgOut, flipType=False) # # # Create a blank canvas for the keyboard # # keyboard_canvas = np.zeros_like(img) # # buttonList = [] # # # Create buttons for the virtual keyboard based on the keys list # # for key in keys[0]: # # buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key)) # # for key in keys[1]: # # buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key)) # # for key in keys[2]: # # buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key)) # # # Draw the buttons on the keyboard canvas # # for button in buttonList: # # x, y = button.pos # # cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1) # # cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3) # # # Handle input and gestures from detected hands # # if hands: # # for hand in hands: # # lmList = hand["lmList"] # # if lmList: # # # Get the coordinates of the index finger tip (landmark 8) # # x8, y8 = lmList[8][0], lmList[8][1] # # for button in buttonList: # # bx, by = button.pos # # bw, bh = button.size # # # Check if the index finger is over a button # # if bx < x8 < bx + bw and by < y8 < by + bh: # # # Highlight the button and update the text # # cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1) # # cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3) # # # Update the output text in session_state # # session_state["output_text"] += button.text # # # Corrected return: Create a video frame from the ndarray image # # return av.VideoFrame.from_ndarray(img, format="bgr24") # # Shared state for output text # if "output_text" not in st.session_state: # st.session_state["output_text"] = "" # class Detection(NamedTuple): # label: str # score: float # box: np.ndarray # @st.cache_resource # Cache label colors # def generate_label_colors(): # return np.random.uniform(0, 255, size=(2, 3)) # Two classes: Left and Right Hand # COLORS = generate_label_colors() # # Initialize MediaPipe Hands # mp_hands = mp.solutions.hands # detector = mp_hands.Hands(static_image_mode=False, max_num_hands=2, min_detection_confidence=0.5) # # Session-specific caching # result_queue: "queue.Queue[List[Detection]]" = queue.Queue() # # Hand detection callback # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: # image = frame.to_ndarray(format="bgr24") # h, w = image.shape[:2] # # Process image with MediaPipe Hands # results = detector.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) # detections = [] # if results.multi_hand_landmarks: # for hand_landmarks, hand_class in zip(results.multi_hand_landmarks, results.multi_handedness): # # Extract bounding box # x_min, y_min = 1, 1 # x_max, y_max = 0, 0 # for lm in hand_landmarks.landmark: # x_min = min(x_min, lm.x) # y_min = min(y_min, lm.y) # x_max = max(x_max, lm.x) # y_max = max(y_max, lm.y) # # Scale bbox to image size # box = np.array([x_min * w, y_min * h, x_max * w, y_max * h]).astype("int") # # Label and score # label = hand_class.classification[0].label # score = hand_class.classification[0].score # detections.append(Detection(label=label, score=score, box=box)) # # Draw bounding box and label # color = COLORS[0 if label == "Left" else 1] # cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), color, 2) # caption = f"{label}: {round(score * 100, 2)}%" # cv2.putText( # image, # caption, # (box[0], box[1] - 15 if box[1] - 15 > 15 else box[1] + 15), # cv2.FONT_HERSHEY_SIMPLEX, # 0.5, # color, # 2, # ) # # Put results in the queue # result_queue.put(detections) # return av.VideoFrame.from_ndarray(image, format="bgr24") # webrtc_ctx = webrtc_streamer( # key="keyboard-demo", # mode=WebRtcMode.SENDRECV, # rtc_configuration={ # "iceServers": get_ice_servers(), # "iceTransportPolicy": "relay", # }, # video_frame_callback=video_frame_callback, # media_stream_constraints={"video": True, "audio": False}, # async_processing=True, # ) # st.markdown("### Instructions") # st.write( # """ # 1. Turn on your webcam using the checkbox above. # 2. Use hand gestures to interact with the virtual keyboard. # """ # ) #) import logging import cv2 import numpy as np import mediapipe as mp import streamlit as st from streamlit_webrtc import webrtc_streamer import av import queue from typing import List # Logging setup logger = logging.getLogger(__name__) # Streamlit setup st.title("AI Squat Detection using WebRTC") st.info("Use your webcam for real-time squat detection.") # Initialize MediaPipe components mp_pose = mp.solutions.pose mp_drawing = mp.solutions.drawing_utils # Angle calculation function def calculate_angle(a, b, c): a = np.array(a) b = np.array(b) c = np.array(c) radians = np.arctan2(c[1]-b[1], c[0]-b[0]) - np.arctan2(a[1]-b[1], a[0]-b[0]) angle = np.abs(radians * 180.0 / np.pi) if angle > 180.0: angle = 360 - angle return angle # Detection Queue result_queue: queue.Queue[List[Detection]] = queue.Queue() def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: image = frame.to_ndarray(format="bgr24") image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) with mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5) as pose: results = pose.process(image_rgb) landmarks = results.pose_landmarks.landmark if results.pose_landmarks else [] # Corrected detection logic detections = [ Detection( class_id=0, # Assuming a generic class_id for pose detections label="Pose", score=1.0, # Full confidence as pose landmarks were detected box=np.array([0, 0, image.shape[1], image.shape[0]]) # Full image as bounding box ) ] if landmarks else [] if landmarks: hip = [landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].x, landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].y] knee = [landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].x, landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].y] ankle = [landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].x, landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].y] shoulder = [landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].x, landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].y] foot = [landmarks[mp_pose.PoseLandmark.LEFT_FOOT_INDEX.value].x, landmarks[mp_pose.PoseLandmark.LEFT_FOOT_INDEX.value].y] # Calculate angles knee_angle = calculate_angle(hip, knee, ankle) hip_angle = calculate_angle(shoulder, hip, [hip[0], 0]) ankle_angle = calculate_angle(foot, ankle, knee) # Display key angles cv2.putText(image, f"Knee: {int(knee_angle)}", (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2) cv2.putText(image, f"Hip: {int(hip_angle)}", (10, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2) cv2.putText(image, f"Ankle: {int(ankle_angle)}", (10, 150), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2) # Squat logic if 80 < knee_angle < 110 and 29 < hip_angle < 40: cv2.putText(image, "Squat Detected!", (300, 100), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 255, 0), 3) else: if hip_angle < 29: cv2.putText(image, "Lean Forward!", (300, 200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3) elif hip_angle > 45: cv2.putText(image, "Lean Backward!", (300, 200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3) if knee_angle < 80: cv2.putText(image, "Squat Too Deep!", (300, 250), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3) elif knee_angle > 110: cv2.putText(image, "Lower Your Hips!", (300, 300), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 3) mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS, mp_drawing.DrawingSpec(color=(255, 175, 0), thickness=2, circle_radius=2), mp_drawing.DrawingSpec(color=(0, 255, 200), thickness=2, circle_radius=2)) result_queue.put(detections) return av.VideoFrame.from_ndarray(image, format="bgr24") # WebRTC streamer configuration webrtc_streamer( key="squat-detection", video_frame_callback=video_frame_callback, media_stream_constraints={"video": True, "audio": False}, async_processing=True ) # import logging # import cv2 # import numpy as np # import streamlit as st # from streamlit_webrtc import WebRtcMode, webrtc_streamer # from cvzone.HandTrackingModule import HandDetector # from cvzone.SelfiSegmentationModule import SelfiSegmentation # import os # import time # import av # import queue # from typing import List, NamedTuple # from sample_utils.turn import get_ice_servers # logger = logging.getLogger(__name__) # # Streamlit settings # st.set_page_config(page_title="Virtual Keyboard", layout="wide") # st.title("Interactive Virtual Keyboard") # st.subheader('''Turn on the webcam and use hand gestures to interact with the virtual keyboard. # Use 'a' and 'd' from the keyboard to change the background.''') # # Initialize modules # detector = HandDetector(maxHands=1, detectionCon=0.8) # segmentor = SelfiSegmentation() # # Define virtual keyboard layout # keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"], # ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"], # ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]] # class Button: # def __init__(self, pos, text, size=[100, 100]): # self.pos = pos # self.size = size # self.text = text # class Detection(NamedTuple): # label: str # score: float # box: np.ndarray # # result_queue: "queue.Queue[List[Detection]]" = queue.Queue() # listImg = os.listdir('model/street') if os.path.exists('model/street') else [] # if not listImg: # st.error("Error: 'street' directory is missing or empty. Please add background images.") # st.stop() # else: # imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg if cv2.imread(f'model/street/{imgPath}') is not None] # indexImg = 0 # prev_key_time = [time.time()] * 2 # output_text = "" # if "output_text" not in st.session_state: # st.session_state["output_text"] = "" # # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: # # img = frame.to_ndarray(format="bgr24") # # hands, img = detector.findHands(img, flipType=False) # # # Render hand detection results # # if hands: # # hand = hands[0] # # bbox = hand["bbox"] # # cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[0]+bbox[2], bbox[1]+bbox[3]), (255, 0, 0), 2) # # cv2.putText(img, 'OpenCV', (50,50), font, # # fontScale, color, thickness, cv2.LINE_AA) # # cv2.putText(img, 'OpenCV', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 1, cv2.LINE_AA) # # result_queue.put(hands) # # return av.VideoFrame.from_ndarray(img, format="bgr24") # result_queue: "queue.Queue[List[Detection]]" = queue.Queue() # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: # image = frame.to_ndarray(format="bgr24") # # Run inference # blob = cv2.dnn.blobFromImage( # cv2.resize(image, (300, 300)), 0.007843, (300, 300), 127.5 # ) # net.setInput(blob) # output = net.forward() # h, w = image.shape[:2] # # Convert the output array into a structured form. # output = output.squeeze() # (1, 1, N, 7) -> (N, 7) # output = output[output[:, 2] >= score_threshold] # detections = [ # Detection( # class_id=int(detection[1]), # label=CLASSES[int(detection[1])], # score=float(detection[2]), # box=(detection[3:7] * np.array([w, h, w, h])), # ) # for detection in output # ] # # Render bounding boxes and captions # for detection in detections: # caption = f"{detection.label}: {round(detection.score * 100, 2)}%" # color = COLORS[detection.class_id] # xmin, ymin, xmax, ymax = detection.box.astype("int") # cv2.rectangle(image, (xmin, ymin), (xmax, ymax), color, 2) # cv2.putText( # image, # caption, # (xmin, ymin - 15 if ymin - 15 > 15 else ymin + 15), # cv2.FONT_HERSHEY_SIMPLEX, # 0.5, # color, # 2, # ) # result_queue.put(detections) # return av.VideoFrame.from_ndarray(image, format="bgr24") # # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: # # global indexImg, output_text # # img = frame.to_ndarray(format="bgr24") # # imgOut = segmentor.removeBG(img, imgList[indexImg]) # # hands, imgOut = detector.findHands(imgOut, flipType=False) # # buttonList = [Button([30 + col * 105, 30 + row * 120], key) for row, line in enumerate(keys) for col, key in enumerate(line)] # # detections = [] # # if hands: # # for i, hand in enumerate(hands): # # lmList = hand['lmList'] # # bbox = hand['bbox'] # # label = "Hand" # # score = hand['score'] # # box = np.array([bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]]) # # detections.append(Detection(label=label, score=score, box=box)) # # if lmList: # # x4, y4 = lmList[4][0], lmList[4][1] # # x8, y8 = lmList[8][0], lmList[8][1] # # distance = np.sqrt((x8 - x4) ** 2 + (y8 - y4) ** 2) # # click_threshold = 10 # # for button in buttonList: # # x, y = button.pos # # w, h = button.size # # if x < x8 < x + w and y < y8 < y + h: # # cv2.rectangle(imgOut, button.pos, (x + w, y + h), (0, 255, 160), -1) # # cv2.putText(imgOut, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3) # # if (distance / np.sqrt(bbox[2] ** 2 + bbox[3] ** 2)) * 100 < click_threshold: # # if time.time() - prev_key_time[i] > 2: # # prev_key_time[i] = time.time() # # if button.text != 'BS' and button.text != 'SPACE': # # output_text += button.text # # elif button.text == 'BS': # # output_text = output_text[:-1] # # else: # # output_text += ' ' # # result_queue.put(detections) # # st.session_state["output_text"] = output_text # # return av.VideoFrame.from_ndarray(imgOut, format="bgr24") # webrtc_streamer( # key="virtual-keyboard", # mode=WebRtcMode.SENDRECV, # rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"}, # media_stream_constraints={"video": True, "audio": False}, # video_frame_callback=video_frame_callback, # async_processing=True, # ) # st.subheader("Output Text") # st.text_area("Live Input:", value=st.session_state["output_text"], height=200)