File size: 7,316 Bytes
0ac8362 c286acb 0ac8362 0325cdc 79ac659 c286acb 0ac8362 94546d9 0ac8362 a03b10d ac79aff a03b10d ac79aff a03b10d ac79aff a03b10d ac79aff a03b10d 94546d9 5b6aa72 4b07573 2a5dacb 9289f95 4600684 4b07573 4600684 4b07573 4600684 5b6aa72 48723b8 a01685d 94546d9 a01685d 8201911 a01685d 0ac8362 94546d9 a03b10d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 |
import logging
import queue
from pathlib import Path
from typing import List, NamedTuple
import av
import cv2
import numpy as np
import streamlit as st
from streamlit_webrtc import WebRtcMode, webrtc_streamer
from sample_utils.turn import get_ice_servers
from cvzone.HandTrackingModule import HandDetector
from cvzone.SelfiSegmentationModule import SelfiSegmentation
import time
import os
logger = logging.getLogger(__name__)
st.title("Interactive Virtual Keyboard with Twilio Integration")
st.info("Use your webcam to interact with the virtual keyboard via hand gestures.")
class Button:
def __init__(self, pos, text, size=[100, 100]):
self.pos = pos
self.size = size
self.text = text
# Function to process the video frame from the webcam
# def process_video_frame(frame: av.VideoFrame, detector, segmentor, imgList, indexImg, keys, session_state)-> av.VideoFrame:
# # Convert the frame to a numpy array (BGR format)
# image = frame.to_ndarray(format="bgr24")
# # Remove background using SelfiSegmentation
# imgOut = segmentor.removeBG(image, imgList[indexImg])
# # Detect hands on the background-removed image
# hands, img = detector.findHands(imgOut, flipType=False)
# # Create a blank canvas for the keyboard
# keyboard_canvas = np.zeros_like(img)
# buttonList = []
# # Create buttons for the virtual keyboard based on the keys list
# for key in keys[0]:
# buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
# for key in keys[1]:
# buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
# for key in keys[2]:
# buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))
# # Draw the buttons on the keyboard canvas
# for button in buttonList:
# x, y = button.pos
# cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
# cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)
# # Handle input and gestures from detected hands
# if hands:
# for hand in hands:
# lmList = hand["lmList"]
# if lmList:
# # Get the coordinates of the index finger tip (landmark 8)
# x8, y8 = lmList[8][0], lmList[8][1]
# for button in buttonList:
# bx, by = button.pos
# bw, bh = button.size
# # Check if the index finger is over a button
# if bx < x8 < bx + bw and by < y8 < by + bh:
# # Highlight the button and update the text
# cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
# cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
# # Update the output text in session_state
# session_state["output_text"] += button.text
# # Corrected return: Create a video frame from the ndarray image
# return av.VideoFrame.from_ndarray(img, format="bgr24")
# Initialize components
detector = HandDetector(maxHands=1, detectionCon=0.8)
segmentor = SelfiSegmentation()
keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]
listImg = os.listdir('model/street')
imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg]
indexImg = 0
# Function to process the video frame from the webcam
def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state):
# Convert the frame to a numpy array (BGR format)
image = frame.to_ndarray(format="bgr24")
# Remove background using SelfiSegmentation
imgOut = segmentor.removeBG(image, imgList[indexImg])
# Detect hands on the background-removed image
hands, img = detector.findHands(imgOut, flipType=False)
# Create a blank canvas for the keyboard
keyboard_canvas = np.zeros_like(img)
buttonList = []
# Create buttons for the virtual keyboard based on the keys list
for key in keys[0]:
buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
for key in keys[1]:
buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
for key in keys[2]:
buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))
# Draw the buttons on the keyboard canvas
for button in buttonList:
x, y = button.pos
cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)
# Handle input and gestures from detected hands
if hands:
for hand in hands:
lmList = hand["lmList"]
if lmList:
# Get the coordinates of the index finger tip (landmark 8)
x8, y8 = lmList[8][0], lmList[8][1]
for button in buttonList:
bx, by = button.pos
bw, bh = button.size
# Check if the index finger is over a button
if bx < x8 < bx + bw and by < y8 < by + bh:
# Highlight the button and update the text
cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
# Update the output text in session_state
session_state["output_text"] += button.text
# Corrected return: Create a video frame from the ndarray image
return av.VideoFrame.from_ndarray(img, format="bgr24")
# def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
# img = frame.to_ndarray(format="bgr24")
# hands, img = detector.findHands(img, flipType=False)
# # Render hand detection results
# if hands:
# hand = hands[0]
# bbox = hand["bbox"]
# cv2.rectangle(img, (bbox[0], bbox[1]), (255, 0, 0), 2)
# return av.VideoFrame.from_ndarray(img, format="bgr24")
# Shared state for output text
if "output_text" not in st.session_state:
st.session_state["output_text"] = ""
# Create a thread-safe queue for passing results from callback
result_queue = queue.Queue()
def video_frame_callback(frame):
# Process the frame asynchronously
processed_frame = process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, st.session_state)
# Put the processed frame into the queue
result_queue.put(processed_frame)
return processed_frame
webrtc_ctx = webrtc_streamer(
key="keyboard-demo",
mode=WebRtcMode.SENDRECV,
rtc_configuration={
"iceServers": get_ice_servers(),
},
video_frame_callback=video_frame_callback,
media_stream_constraints={"video": True, "audio": False},
)
st.markdown("### Instructions")
st.write(
"""
1. Turn on your webcam using the checkbox above.
2. Use hand gestures to interact with the virtual keyboard.
"""
)
|