| import logging | |
| import queue | |
| from pathlib import Path | |
| from typing import List, NamedTuple | |
| import av | |
| import cv2 | |
| import numpy as np | |
| import streamlit as st | |
| from streamlit_webrtc import WebRtcMode, webrtc_streamer | |
| from sample_utils.turn import get_ice_servers | |
| from cvzone.HandTrackingModule import HandDetector | |
| from cvzone.SelfiSegmentationModule import SelfiSegmentation | |
| import time | |
| import os | |
| logger = logging.getLogger(__name__) | |
| st.title("Interactive Virtual Keyboard with Twilio Integration") | |
| st.info("Use your webcam to interact with the virtual keyboard via hand gestures.") | |
| class Button: | |
| def __init__(self, pos, text, size=[100, 100]): | |
| self.pos = pos | |
| self.size = size | |
| self.text = text | |
| # Function to process the video frame from the webcam | |
| # def process_video_frame(frame: av.VideoFrame, detector, segmentor, imgList, indexImg, keys, session_state)-> av.VideoFrame: | |
| # # Convert the frame to a numpy array (BGR format) | |
| # image = frame.to_ndarray(format="bgr24") | |
| # # Remove background using SelfiSegmentation | |
| # imgOut = segmentor.removeBG(image, imgList[indexImg]) | |
| # # Detect hands on the background-removed image | |
| # hands, img = detector.findHands(imgOut, flipType=False) | |
| # # Create a blank canvas for the keyboard | |
| # keyboard_canvas = np.zeros_like(img) | |
| # buttonList = [] | |
| # # Create buttons for the virtual keyboard based on the keys list | |
| # for key in keys[0]: | |
| # buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key)) | |
| # for key in keys[1]: | |
| # buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key)) | |
| # for key in keys[2]: | |
| # buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key)) | |
| # # Draw the buttons on the keyboard canvas | |
| # for button in buttonList: | |
| # x, y = button.pos | |
| # cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1) | |
| # cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3) | |
| # # Handle input and gestures from detected hands | |
| # if hands: | |
| # for hand in hands: | |
| # lmList = hand["lmList"] | |
| # if lmList: | |
| # # Get the coordinates of the index finger tip (landmark 8) | |
| # x8, y8 = lmList[8][0], lmList[8][1] | |
| # for button in buttonList: | |
| # bx, by = button.pos | |
| # bw, bh = button.size | |
| # # Check if the index finger is over a button | |
| # if bx < x8 < bx + bw and by < y8 < by + bh: | |
| # # Highlight the button and update the text | |
| # cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1) | |
| # cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3) | |
| # # Update the output text in session_state | |
| # session_state["output_text"] += button.text | |
| # # Corrected return: Create a video frame from the ndarray image | |
| # return av.VideoFrame.from_ndarray(img, format="bgr24") | |
| # Initialize components | |
| detector = HandDetector(maxHands=1, detectionCon=0.8) | |
| segmentor = SelfiSegmentation() | |
| keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"], | |
| ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"], | |
| ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]] | |
| listImg = os.listdir('model/street') | |
| imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg] | |
| indexImg = 0 | |
| # Function to process the video frame from the webcam | |
| def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state): | |
| # Convert the frame to a numpy array (BGR format) | |
| image = frame.to_ndarray(format="bgr24") | |
| # Remove background using SelfiSegmentation | |
| imgOut = segmentor.removeBG(image, imgList[indexImg]) | |
| # Detect hands on the background-removed image | |
| hands, img = detector.findHands(imgOut, flipType=False) | |
| # Create a blank canvas for the keyboard | |
| keyboard_canvas = np.zeros_like(img) | |
| buttonList = [] | |
| # Create buttons for the virtual keyboard based on the keys list | |
| for key in keys[0]: | |
| buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key)) | |
| for key in keys[1]: | |
| buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key)) | |
| for key in keys[2]: | |
| buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key)) | |
| # Draw the buttons on the keyboard canvas | |
| for button in buttonList: | |
| x, y = button.pos | |
| cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1) | |
| cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3) | |
| # Handle input and gestures from detected hands | |
| if hands: | |
| for hand in hands: | |
| lmList = hand["lmList"] | |
| if lmList: | |
| # Get the coordinates of the index finger tip (landmark 8) | |
| x8, y8 = lmList[8][0], lmList[8][1] | |
| for button in buttonList: | |
| bx, by = button.pos | |
| bw, bh = button.size | |
| # Check if the index finger is over a button | |
| if bx < x8 < bx + bw and by < y8 < by + bh: | |
| # Highlight the button and update the text | |
| cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1) | |
| cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3) | |
| # Update the output text in session_state | |
| session_state["output_text"] += button.text | |
| # Corrected return: Create a video frame from the ndarray image | |
| return av.VideoFrame.from_ndarray(img, format="bgr24") | |
| # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
| # img = frame.to_ndarray(format="bgr24") | |
| # hands, img = detector.findHands(img, flipType=False) | |
| # # Render hand detection results | |
| # if hands: | |
| # hand = hands[0] | |
| # bbox = hand["bbox"] | |
| # cv2.rectangle(img, (bbox[0], bbox[1]), (255, 0, 0), 2) | |
| # return av.VideoFrame.from_ndarray(img, format="bgr24") | |
| # Shared state for output text | |
| if "output_text" not in st.session_state: | |
| st.session_state["output_text"] = "" | |
| # Create a thread-safe queue for passing results from callback | |
| result_queue = queue.Queue() | |
| def video_frame_callback(frame): | |
| # Process the frame asynchronously | |
| processed_frame = process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, st.session_state) | |
| # Put the processed frame into the queue | |
| result_queue.put(processed_frame) | |
| return processed_frame | |
| webrtc_ctx = webrtc_streamer( | |
| key="keyboard-demo", | |
| mode=WebRtcMode.SENDRECV, | |
| rtc_configuration={ | |
| "iceServers": get_ice_servers(), | |
| }, | |
| video_frame_callback=video_frame_callback, | |
| media_stream_constraints={"video": True, "audio": False}, | |
| ) | |
| st.markdown("### Instructions") | |
| st.write( | |
| """ | |
| 1. Turn on your webcam using the checkbox above. | |
| 2. Use hand gestures to interact with the virtual keyboard. | |
| """ | |
| ) | |