File size: 2,749 Bytes
8083389
 
 
 
 
 
 
 
 
 
 
 
 
20e2e7b
 
8083389
 
84c826d
8083389
 
 
 
91368dd
51bb045
91368dd
 
 
 
 
 
 
 
 
 
 
 
 
51bb045
91368dd
 
 
 
 
20e2e7b
 
91368dd
 
 
 
 
 
 
20e2e7b
91368dd
 
 
51bb045
91368dd
51bb045
 
20e2e7b
91368dd
 
 
51bb045
20e2e7b
91368dd
 
 
 
20e2e7b
51bb045
 
20e2e7b
91368dd
 
84c826d
91368dd
20e2e7b
91368dd
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
import logging
import queue
from typing import List, NamedTuple
import av
import cv2
import numpy as np
import streamlit as st
from streamlit_webrtc import WebRtcMode, webrtc_streamer
from sample_utils.turn import get_ice_servers
from cvzone.HandTrackingModule import HandDetector
import os
import time

# Logger Setup
logger = logging.getLogger(__name__)

# Streamlit settings
st.set_page_config(page_title="Virtual Keyboard", page_icon="🏋️")
st.title("Interactive Virtual Keyboard")
st.subheader('''Turn on the webcam and use hand gestures to interact with the virtual keyboard.
Use 'a' and 'd' from the keyboard to change the background.''')

# Initialize modules
detector = HandDetector(maxHands=1, detectionCon=0.7)

# Define virtual keyboard layout
keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
        ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
        ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]

class Detection(NamedTuple):
    label: str
    score: float
    box: np.ndarray

result_queue: "queue.Queue[List[Detection]]" = queue.Queue()

# Background image loading
listImg = os.listdir('model/street') if os.path.exists('model/street') else []
if not listImg:
    st.error("Error: 'street' directory is missing or empty. Please add background images.")
    st.stop()
else:
    imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg]
    imgList = [img for img in imgList if img is not None]

indexImg = 0
output_text = ""

if "output_text" not in st.session_state:
    st.session_state["output_text"] = ""

# Video Frame Callback
def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
    global indexImg, output_text

    # Convert the frame to BGR
    img = frame.to_ndarray(format="bgr24")

    # Process the frame with Hand Detector
    hands, img = detector.findHands(img, draw=True)

    detections = []
    if hands:
        logger.info(f"Detected {len(hands)} hand(s).")
        for hand in hands:
            bbox = hand['bbox']
            label = "Hand"
            score = hand['score']
            box = np.array([bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]])
            detections.append(Detection(label=label, score=score, box=box))
    else:
        logger.info("No hands detected.")

    result_queue.put(detections)
    st.session_state["output_text"] = output_text
    return av.VideoFrame.from_ndarray(img, format="bgr24")

# WebRTC Streamer
webrtc_streamer(
    key="virtual-keyboard",
    mode=WebRtcMode.SENDRECV,
    rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"},
    media_stream_constraints={"video": True, "audio": False},
    video_frame_callback=video_frame_callback,
    async_processing=True,
)