File size: 2,950 Bytes
8083389 54cabae 8083389 84c826d 8083389 91368dd 84c826d 91368dd 84c826d 91368dd d181aba 91368dd 84c826d 91368dd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 |
import logging
import queue
from pathlib import Path
from typing import List, NamedTuple
import mediapipe as mp
import av
import cv2
import numpy as np
import streamlit as st
from streamlit_webrtc import WebRtcMode, webrtc_streamer
from sample_utils.download import download_file
from sample_utils.turn import get_ice_servers
from cvzone.HandTrackingModule import HandDetector
from cvzone.SelfiSegmentationModule import SelfiSegmentation
import os
import time
# logger = logging.getLogger(_name_)
# Streamlit settings
st.set_page_config(page_title="Virtual Keyboard", page_icon="🏋️")
st.title("Interactive Virtual Keyboard")
st.subheader('''Turn on the webcam and use hand gestures to interact with the virtual keyboard.
Use 'a' and 'd' from the keyboard to change the background.''')
# Initialize modules
detector = HandDetector(maxHands=1, detectionCon=0.8)
segmentor = SelfiSegmentation()
# Define virtual keyboard layout
keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]
class Button:
def __init__(self, pos, text, size=[100, 100]):
self.pos = pos
self.size = size
self.text = text
class Detection(NamedTuple):
label: str
score: float
box: np.ndarray
result_queue: "queue.Queue[List[Detection]]" = queue.Queue()
listImg = os.listdir('model/street') if os.path.exists('model/street') else []
if not listImg:
st.error("Error: 'street' directory is missing or empty. Please add background images.")
st.stop()
else:
imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg if cv2.imread(f'model/street/{imgPath}') is not None]
indexImg = 0
prev_key_time = [time.time()] * 2
output_text = ""
if "output_text" not in st.session_state:
st.session_state["output_text"] = ""
def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
global indexImg, output_text
img = frame.to_ndarray(format="bgr24")
hands = detector.findHands(img, draw=False)
detections = []
if hands:
for i, hand in enumerate(hands):
lmList = hand['lmList']
bbox = hand['bbox']
label = "Hand"
score = hand['score']
box = np.array([bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]])
detections.append(Detection(label=label, score=score, box=box))
cv2.imshow('WebCam with Virtual Keyboard', img)
result_queue.put(detections)
st.session_state["output_text"] = output_text
return av.VideoFrame.from_ndarray(img, format="bgr24")
webrtc_streamer(
key="virtual-keyboard",
mode=WebRtcMode.SENDRECV,
rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"},
media_stream_constraints={"video": True, "audio": False},
video_frame_callback=video_frame_callback,
async_processing=True,
)
|