import logging import queue from pathlib import Path from typing import List, NamedTuple import av import cv2 import numpy as np import streamlit as st from streamlit_webrtc import WebRtcMode, webrtc_streamer from sample_utils.turn import get_ice_servers from cvzone.HandTrackingModule import HandDetector from cvzone.SelfiSegmentationModule import SelfiSegmentation import time import os logger = logging.getLogger(__name__) st.title("Interactive Virtual Keyboard with Twilio Integration") st.info("Use your webcam to interact with the virtual keyboard via hand gestures.") class Button: def __init__(self, pos, text, size=[100, 100]): self.pos = pos self.size = size self.text = text # Initialize components detector = HandDetector(maxHands=1, detectionCon=0.8) # segmentor = SelfiSegmentation() # keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"], # ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"], # ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]] # listImg = os.listdir('model/street') # imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg] # indexImg = 0 # # Function to process the video frame from the webcam # def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state): # # Convert the frame to a numpy array (BGR format) # image = frame.to_ndarray(format="bgr24") # # Remove background using SelfiSegmentation # imgOut = segmentor.removeBG(image, imgList[indexImg]) # # Detect hands on the background-removed image # hands, img = detector.findHands(imgOut, flipType=False) # # Create a blank canvas for the keyboard # keyboard_canvas = np.zeros_like(img) # buttonList = [] # # Create buttons for the virtual keyboard based on the keys list # for key in keys[0]: # buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key)) # for key in keys[1]: # buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key)) # for key in keys[2]: # buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key)) # # Draw the buttons on the keyboard canvas # for button in buttonList: # x, y = button.pos # cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1) # cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3) # # Handle input and gestures from detected hands # if hands: # for hand in hands: # lmList = hand["lmList"] # if lmList: # # Get the coordinates of the index finger tip (landmark 8) # x8, y8 = lmList[8][0], lmList[8][1] # for button in buttonList: # bx, by = button.pos # bw, bh = button.size # # Check if the index finger is over a button # if bx < x8 < bx + bw and by < y8 < by + bh: # # Highlight the button and update the text # cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1) # cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3) # # Update the output text in session_state # session_state["output_text"] += button.text # # Corrected return: Create a video frame from the ndarray image # return av.VideoFrame.from_ndarray(img, format="bgr24") # Shared state for output text if "output_text" not in st.session_state: st.session_state["output_text"] = "" class Detection(NamedTuple): label: str score: float box: np.ndarray @st.cache_resource # Cache label colors def generate_label_colors(): return np.random.uniform(0, 255, size=(2, 3)) # Two classes: Left and Right Hand COLORS = generate_label_colors() # Initialize MediaPipe Hands mp_hands = mp.solutions.hands detector = mp_hands.Hands(static_image_mode=False, max_num_hands=2, min_detection_confidence=0.5) # Session-specific caching result_queue: "queue.Queue[List[Detection]]" = queue.Queue() # Hand detection callback def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: image = frame.to_ndarray(format="bgr24") h, w = image.shape[:2] # Process image with MediaPipe Hands results = detector.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) detections = [] if results.multi_hand_landmarks: for hand_landmarks, hand_class in zip(results.multi_hand_landmarks, results.multi_handedness): # Extract bounding box x_min, y_min = 1, 1 x_max, y_max = 0, 0 for lm in hand_landmarks.landmark: x_min = min(x_min, lm.x) y_min = min(y_min, lm.y) x_max = max(x_max, lm.x) y_max = max(y_max, lm.y) # Scale bbox to image size box = np.array([x_min * w, y_min * h, x_max * w, y_max * h]).astype("int") # Label and score label = hand_class.classification[0].label score = hand_class.classification[0].score detections.append(Detection(label=label, score=score, box=box)) # Draw bounding box and label color = COLORS[0 if label == "Left" else 1] cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), color, 2) caption = f"{label}: {round(score * 100, 2)}%" cv2.putText( image, caption, (box[0], box[1] - 15 if box[1] - 15 > 15 else box[1] + 15), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2, ) # Put results in the queue result_queue.put(detections) return av.VideoFrame.from_ndarray(image, format="bgr24") webrtc_ctx = webrtc_streamer( key="keyboard-demo", mode=WebRtcMode.SENDRECV, rtc_configuration={ "iceServers": get_ice_servers(), "iceTransportPolicy": "relay", }, video_frame_callback=video_frame_callback, media_stream_constraints={"video": True, "audio": False}, async_processing=True, ) st.markdown("### Instructions") st.write( """ 1. Turn on your webcam using the checkbox above. 2. Use hand gestures to interact with the virtual keyboard. """ )