File size: 4,529 Bytes
0ac8362 c286acb 0ac8362 0325cdc 79ac659 c286acb 0ac8362 94546d9 0ac8362 a03b10d 94546d9 5b6aa72 a03b10d a01685d 94546d9 a01685d 8201911 a01685d 8201911 a01685d 0ac8362 94546d9 a03b10d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 |
import logging
import queue
from pathlib import Path
from typing import List, NamedTuple
import av
import cv2
import numpy as np
import streamlit as st
from streamlit_webrtc import WebRtcMode, webrtc_streamer
from sample_utils.turn import get_ice_servers
from cvzone.HandTrackingModule import HandDetector
from cvzone.SelfiSegmentationModule import SelfiSegmentation
import time
import os
logger = logging.getLogger(__name__)
st.title("Interactive Virtual Keyboard with Twilio Integration")
st.info("Use your webcam to interact with the virtual keyboard via hand gestures.")
class Button:
def __init__(self, pos, text, size=[100, 100]):
self.pos = pos
self.size = size
self.text = text
# Function to process the video frame from the webcam
def process_video_frame(frame: av.VideoFrame, detector, segmentor, imgList, indexImg, keys, session_state)-> av.VideoFrame:
# Convert the frame to a numpy array (BGR format)
image = frame.to_ndarray(format="bgr24")
# Remove background using SelfiSegmentation
imgOut = segmentor.removeBG(image, imgList[indexImg])
# Detect hands on the background-removed image
hands, img = detector.findHands(imgOut, flipType=False)
# Create a blank canvas for the keyboard
keyboard_canvas = np.zeros_like(img)
buttonList = []
# Create buttons for the virtual keyboard based on the keys list
for key in keys[0]:
buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
for key in keys[1]:
buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
for key in keys[2]:
buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))
# Draw the buttons on the keyboard canvas
for button in buttonList:
x, y = button.pos
cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)
# Handle input and gestures from detected hands
if hands:
for hand in hands:
lmList = hand["lmList"]
if lmList:
# Get the coordinates of the index finger tip (landmark 8)
x8, y8 = lmList[8][0], lmList[8][1]
for button in buttonList:
bx, by = button.pos
bw, bh = button.size
# Check if the index finger is over a button
if bx < x8 < bx + bw and by < y8 < by + bh:
# Highlight the button and update the text
cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
# Update the output text in session_state
session_state["output_text"] += button.text
# Corrected return: Create a video frame from the ndarray image
return av.VideoFrame.from_ndarray(img, format="bgr24")
# Initialize components
detector = HandDetector(maxHands=1, detectionCon=0.8)
segmentor = SelfiSegmentation()
keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]
listImg = os.listdir('model/street')
imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg]
indexImg = 0
# Shared state for output text
if "output_text" not in st.session_state:
st.session_state["output_text"] = ""
# Create a thread-safe queue for passing results from callback
result_queue = queue.Queue()
def video_frame_callback(frame):
# Process the frame asynchronously
processed_frame = process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, st.session_state)
# Put the processed frame into the queue
result_queue.put(processed_frame)
return processed_frame
webrtc_ctx = webrtc_streamer(
key="keyboard-demo",
mode=WebRtcMode.SENDRECV,
rtc_configuration={
"iceServers": get_ice_servers(),
"iceTransportPolicy": "relay",
},
video_frame_callback=video_frame_callback,
media_stream_constraints={"video": True, "audio": False},
async_processing=True,
)
st.markdown("### Instructions")
st.write(
"""
1. Turn on your webcam using the checkbox above.
2. Use hand gestures to interact with the virtual keyboard.
"""
)
|