import logging import queue from pathlib import Path from typing import List, NamedTuple import mediapipe as mp import av import cv2 import numpy as np import streamlit as st from streamlit_webrtc import WebRtcMode, webrtc_streamer from sample_utils.turn import get_ice_servers from cvzone.HandTrackingModule import HandDetector from cvzone.SelfiSegmentationModule import SelfiSegmentation import time import os logger = logging.getLogger(__name__) st.title("Interactive Virtual Keyboard with Twilio Integration") st.info("Use your webcam to interact with the virtual keyboard via hand gestures.") class Button: def __init__(self, pos, text, size=[100, 100]): self.pos = pos self.size = size self.text = text # Initialize components detector = HandDetector(maxHands=1, detectionCon=0.8) # segmentor = SelfiSegmentation() # keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"], # ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"], # ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]] # listImg = os.listdir('model/street') # imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg] # indexImg = 0 # # Function to process the video frame from the webcam # def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state): # # Convert the frame to a numpy array (BGR format) # image = frame.to_ndarray(format="bgr24") # # Remove background using SelfiSegmentation # imgOut = segmentor.removeBG(image, imgList[indexImg]) # # Detect hands on the background-removed image # hands, img = detector.findHands(imgOut, flipType=False) # # Create a blank canvas for the keyboard # keyboard_canvas = np.zeros_like(img) # buttonList = [] # # Create buttons for the virtual keyboard based on the keys list # for key in keys[0]: # buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key)) # for key in keys[1]: # buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key)) # for key in keys[2]: # buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key)) # # Draw the buttons on the keyboard canvas # for button in buttonList: # x, y = button.pos # cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1) # cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3) # # Handle input and gestures from detected hands # if hands: # for hand in hands: # lmList = hand["lmList"] # if lmList: # # Get the coordinates of the index finger tip (landmark 8) # x8, y8 = lmList[8][0], lmList[8][1] # for button in buttonList: # bx, by = button.pos # bw, bh = button.size # # Check if the index finger is over a button # if bx < x8 < bx + bw and by < y8 < by + bh: # # Highlight the button and update the text # cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1) # cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3) # # Update the output text in session_state # session_state["output_text"] += button.text # # Corrected return: Create a video frame from the ndarray image # return av.VideoFrame.from_ndarray(img, format="bgr24") # Shared state for output text if "output_text" not in st.session_state: st.session_state["output_text"] = "" class Detection(NamedTuple): label: str score: float box: np.ndarray @st.cache_resource # Cache label colors def generate_label_colors(): return np.random.uniform(0, 255, size=(2, 3)) # Two classes: Left and Right Hand COLORS = generate_label_colors() # Initialize MediaPipe Hands mp_hands = mp.solutions.hands detector = mp_hands.Hands(static_image_mode=False, max_num_hands=2, min_detection_confidence=0.5) # Session-specific caching result_queue: "queue.Queue[List[Detection]]" = queue.Queue() # Hand detection callback def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: image = frame.to_ndarray(format="bgr24") h, w = image.shape[:2] # Process image with MediaPipe Hands results = detector.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) detections = [] if results.multi_hand_landmarks: for hand_landmarks, hand_class in zip(results.multi_hand_landmarks, results.multi_handedness): # Extract bounding box x_min, y_min = 1, 1 x_max, y_max = 0, 0 for lm in hand_landmarks.landmark: x_min = min(x_min, lm.x) y_min = min(y_min, lm.y) x_max = max(x_max, lm.x) y_max = max(y_max, lm.y) # Scale bbox to image size box = np.array([x_min * w, y_min * h, x_max * w, y_max * h]).astype("int") # Label and score label = hand_class.classification[0].label score = hand_class.classification[0].score detections.append(Detection(label=label, score=score, box=box)) # Draw bounding box and label color = COLORS[0 if label == "Left" else 1] cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), color, 2) caption = f"{label}: {round(score * 100, 2)}%" cv2.putText( image, caption, (box[0], box[1] - 15 if box[1] - 15 > 15 else box[1] + 15), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2, ) # Put results in the queue result_queue.put(detections) return av.VideoFrame.from_ndarray(image, format="bgr24") webrtc_ctx = webrtc_streamer( key="keyboard-demo", mode=WebRtcMode.SENDRECV, rtc_configuration={ "iceServers": get_ice_servers(), "iceTransportPolicy": "relay", }, video_frame_callback=video_frame_callback, media_stream_constraints={"video": True, "audio": False}, async_processing=True, ) st.markdown("### Instructions") st.write( """ 1. Turn on your webcam using the checkbox above. 2. Use hand gestures to interact with the virtual keyboard. """ )