| # import logging | |
| # import queue | |
| # from pathlib import Path | |
| # from typing import List, NamedTuple | |
| # import mediapipe as mp | |
| # import av | |
| # import cv2 | |
| # import numpy as np | |
| # import streamlit as st | |
| # from streamlit_webrtc import WebRtcMode, webrtc_streamer | |
| # from sample_utils.turn import get_ice_servers | |
| # from cvzone.HandTrackingModule import HandDetector | |
| # from cvzone.SelfiSegmentationModule import SelfiSegmentation | |
| # import time | |
| # import os | |
| # logger = logging.getLogger(__name__) | |
| # st.title("Interactive Virtual Keyboard with Twilio Integration") | |
| # st.info("Use your webcam to interact with the virtual keyboard via hand gestures.") | |
| # class Button: | |
| # def __init__(self, pos, text, size=[100, 100]): | |
| # self.pos = pos | |
| # self.size = size | |
| # self.text = text | |
| # # Initialize components | |
| # detector = HandDetector(maxHands=1, detectionCon=0.8) | |
| # # segmentor = SelfiSegmentation() | |
| # # keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"], | |
| # # ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"], | |
| # # ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]] | |
| # # listImg = os.listdir('model/street') | |
| # # imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg] | |
| # # indexImg = 0 | |
| # # # Function to process the video frame from the webcam | |
| # # def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state): | |
| # # # Convert the frame to a numpy array (BGR format) | |
| # # image = frame.to_ndarray(format="bgr24") | |
| # # # Remove background using SelfiSegmentation | |
| # # imgOut = segmentor.removeBG(image, imgList[indexImg]) | |
| # # # Detect hands on the background-removed image | |
| # # hands, img = detector.findHands(imgOut, flipType=False) | |
| # # # Create a blank canvas for the keyboard | |
| # # keyboard_canvas = np.zeros_like(img) | |
| # # buttonList = [] | |
| # # # Create buttons for the virtual keyboard based on the keys list | |
| # # for key in keys[0]: | |
| # # buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key)) | |
| # # for key in keys[1]: | |
| # # buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key)) | |
| # # for key in keys[2]: | |
| # # buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key)) | |
| # # # Draw the buttons on the keyboard canvas | |
| # # for button in buttonList: | |
| # # x, y = button.pos | |
| # # cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1) | |
| # # cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3) | |
| # # # Handle input and gestures from detected hands | |
| # # if hands: | |
| # # for hand in hands: | |
| # # lmList = hand["lmList"] | |
| # # if lmList: | |
| # # # Get the coordinates of the index finger tip (landmark 8) | |
| # # x8, y8 = lmList[8][0], lmList[8][1] | |
| # # for button in buttonList: | |
| # # bx, by = button.pos | |
| # # bw, bh = button.size | |
| # # # Check if the index finger is over a button | |
| # # if bx < x8 < bx + bw and by < y8 < by + bh: | |
| # # # Highlight the button and update the text | |
| # # cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1) | |
| # # cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3) | |
| # # # Update the output text in session_state | |
| # # session_state["output_text"] += button.text | |
| # # # Corrected return: Create a video frame from the ndarray image | |
| # # return av.VideoFrame.from_ndarray(img, format="bgr24") | |
| # # Shared state for output text | |
| # if "output_text" not in st.session_state: | |
| # st.session_state["output_text"] = "" | |
| # class Detection(NamedTuple): | |
| # label: str | |
| # score: float | |
| # box: np.ndarray | |
| # @st.cache_resource # Cache label colors | |
| # def generate_label_colors(): | |
| # return np.random.uniform(0, 255, size=(2, 3)) # Two classes: Left and Right Hand | |
| # COLORS = generate_label_colors() | |
| # # Initialize MediaPipe Hands | |
| # mp_hands = mp.solutions.hands | |
| # detector = mp_hands.Hands(static_image_mode=False, max_num_hands=2, min_detection_confidence=0.5) | |
| # # Session-specific caching | |
| # result_queue: "queue.Queue[List[Detection]]" = queue.Queue() | |
| # # Hand detection callback | |
| # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
| # image = frame.to_ndarray(format="bgr24") | |
| # h, w = image.shape[:2] | |
| # # Process image with MediaPipe Hands | |
| # results = detector.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) | |
| # detections = [] | |
| # if results.multi_hand_landmarks: | |
| # for hand_landmarks, hand_class in zip(results.multi_hand_landmarks, results.multi_handedness): | |
| # # Extract bounding box | |
| # x_min, y_min = 1, 1 | |
| # x_max, y_max = 0, 0 | |
| # for lm in hand_landmarks.landmark: | |
| # x_min = min(x_min, lm.x) | |
| # y_min = min(y_min, lm.y) | |
| # x_max = max(x_max, lm.x) | |
| # y_max = max(y_max, lm.y) | |
| # # Scale bbox to image size | |
| # box = np.array([x_min * w, y_min * h, x_max * w, y_max * h]).astype("int") | |
| # # Label and score | |
| # label = hand_class.classification[0].label | |
| # score = hand_class.classification[0].score | |
| # detections.append(Detection(label=label, score=score, box=box)) | |
| # # Draw bounding box and label | |
| # color = COLORS[0 if label == "Left" else 1] | |
| # cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), color, 2) | |
| # caption = f"{label}: {round(score * 100, 2)}%" | |
| # cv2.putText( | |
| # image, | |
| # caption, | |
| # (box[0], box[1] - 15 if box[1] - 15 > 15 else box[1] + 15), | |
| # cv2.FONT_HERSHEY_SIMPLEX, | |
| # 0.5, | |
| # color, | |
| # 2, | |
| # ) | |
| # # Put results in the queue | |
| # result_queue.put(detections) | |
| # return av.VideoFrame.from_ndarray(image, format="bgr24") | |
| # webrtc_ctx = webrtc_streamer( | |
| # key="keyboard-demo", | |
| # mode=WebRtcMode.SENDRECV, | |
| # rtc_configuration={ | |
| # "iceServers": get_ice_servers(), | |
| # "iceTransportPolicy": "relay", | |
| # }, | |
| # video_frame_callback=video_frame_callback, | |
| # media_stream_constraints={"video": True, "audio": False}, | |
| # async_processing=True, | |
| # ) | |
| # st.markdown("### Instructions") | |
| # st.write( | |
| # """ | |
| # 1. Turn on your webcam using the checkbox above. | |
| # 2. Use hand gestures to interact with the virtual keyboard. | |
| # """ | |
| # ) | |
| import logging | |
| import cv2 | |
| import numpy as np | |
| import streamlit as st | |
| from streamlit_webrtc import WebRtcMode, webrtc_streamer | |
| from cvzone.HandTrackingModule import HandDetector | |
| from cvzone.SelfiSegmentationModule import SelfiSegmentation | |
| import os | |
| import time | |
| import av | |
| import queue | |
| from typing import List, NamedTuple | |
| from sample_utils.turn import get_ice_servers | |
| logger = logging.getLogger(__name__) | |
| # Streamlit settings | |
| st.set_page_config(page_title="Virtual Keyboard", layout="wide") | |
| st.title("Interactive Virtual Keyboard") | |
| st.subheader('''Turn on the webcam and use hand gestures to interact with the virtual keyboard. | |
| Use 'a' and 'd' from the keyboard to change the background.''') | |
| # Initialize modules | |
| detector = HandDetector(maxHands=1, detectionCon=0.8) | |
| segmentor = SelfiSegmentation() | |
| # Define virtual keyboard layout | |
| keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"], | |
| ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"], | |
| ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]] | |
| class Button: | |
| def __init__(self, pos, text, size=[100, 100]): | |
| self.pos = pos | |
| self.size = size | |
| self.text = text | |
| class Detection(NamedTuple): | |
| label: str | |
| score: float | |
| box: np.ndarray | |
| # result_queue: "queue.Queue[List[Detection]]" = queue.Queue() | |
| listImg = os.listdir('model/street') if os.path.exists('model/street') else [] | |
| if not listImg: | |
| st.error("Error: 'street' directory is missing or empty. Please add background images.") | |
| st.stop() | |
| else: | |
| imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg if cv2.imread(f'model/street/{imgPath}') is not None] | |
| indexImg = 0 | |
| prev_key_time = [time.time()] * 2 | |
| output_text = "" | |
| if "output_text" not in st.session_state: | |
| st.session_state["output_text"] = "" | |
| # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
| # img = frame.to_ndarray(format="bgr24") | |
| # hands, img = detector.findHands(img, flipType=False) | |
| # # Render hand detection results | |
| # if hands: | |
| # hand = hands[0] | |
| # bbox = hand["bbox"] | |
| # cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[0]+bbox[2], bbox[1]+bbox[3]), (255, 0, 0), 2) | |
| # cv2.putText(img, 'OpenCV', (50,50), font, | |
| # fontScale, color, thickness, cv2.LINE_AA) | |
| # cv2.putText(img, 'OpenCV', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 1, cv2.LINE_AA) | |
| # result_queue.put(hands) | |
| # return av.VideoFrame.from_ndarray(img, format="bgr24") | |
| # Initialize result queue | |
| result_queue = queue.Queue() | |
| def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
| img = frame.to_ndarray(format="bgr24") | |
| # Ensure square input for MediaPipe | |
| h, w, _ = img.shape | |
| size = min(h, w) | |
| img_cropped = img[:size, :size] | |
| # Detect hands | |
| hands, img_cropped = detector.findHands(img_cropped, flipType=False) | |
| # Collect detections | |
| detections = [] | |
| if hands: | |
| for hand in hands: | |
| bbox = hand["bbox"] | |
| label = hand["type"] | |
| score = hand["score"] | |
| # Draw bounding box | |
| cv2.rectangle(img_cropped, (bbox[0], bbox[1]), | |
| (bbox[0] + bbox[2], bbox[1] + bbox[3]), (255, 0, 0), 2) | |
| # Append detection details | |
| detections.append({"label": label, "score": score, "bbox": bbox}) | |
| # Put detections into result queue | |
| result_queue.put(detections) | |
| return av.VideoFrame.from_ndarray(img_cropped, format="bgr24") | |
| # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame: | |
| # global indexImg, output_text | |
| # img = frame.to_ndarray(format="bgr24") | |
| # imgOut = segmentor.removeBG(img, imgList[indexImg]) | |
| # hands, imgOut = detector.findHands(imgOut, flipType=False) | |
| # buttonList = [Button([30 + col * 105, 30 + row * 120], key) for row, line in enumerate(keys) for col, key in enumerate(line)] | |
| # detections = [] | |
| # if hands: | |
| # for i, hand in enumerate(hands): | |
| # lmList = hand['lmList'] | |
| # bbox = hand['bbox'] | |
| # label = "Hand" | |
| # score = hand['score'] | |
| # box = np.array([bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]]) | |
| # detections.append(Detection(label=label, score=score, box=box)) | |
| # if lmList: | |
| # x4, y4 = lmList[4][0], lmList[4][1] | |
| # x8, y8 = lmList[8][0], lmList[8][1] | |
| # distance = np.sqrt((x8 - x4) ** 2 + (y8 - y4) ** 2) | |
| # click_threshold = 10 | |
| # for button in buttonList: | |
| # x, y = button.pos | |
| # w, h = button.size | |
| # if x < x8 < x + w and y < y8 < y + h: | |
| # cv2.rectangle(imgOut, button.pos, (x + w, y + h), (0, 255, 160), -1) | |
| # cv2.putText(imgOut, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3) | |
| # if (distance / np.sqrt(bbox[2] ** 2 + bbox[3] ** 2)) * 100 < click_threshold: | |
| # if time.time() - prev_key_time[i] > 2: | |
| # prev_key_time[i] = time.time() | |
| # if button.text != 'BS' and button.text != 'SPACE': | |
| # output_text += button.text | |
| # elif button.text == 'BS': | |
| # output_text = output_text[:-1] | |
| # else: | |
| # output_text += ' ' | |
| # result_queue.put(detections) | |
| # st.session_state["output_text"] = output_text | |
| # return av.VideoFrame.from_ndarray(imgOut, format="bgr24") | |
| webrtc_streamer( | |
| key="virtual-keyboard", | |
| mode=WebRtcMode.SENDRECV, | |
| rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"}, | |
| media_stream_constraints={"video": True, "audio": False}, | |
| video_frame_callback=video_frame_callback, | |
| async_processing=True, | |
| ) | |
| st.subheader("Output Text") | |
| st.text_area("Live Input:", value=st.session_state["output_text"], height=200) | |