File size: 6,189 Bytes
8083389
 
 
 
 
 
 
 
 
ce9d171
8083389
 
 
20e2e7b
ed1d71f
8083389
 
ed1d71f
8083389
ed1d71f
8083389
fb324e7
ce9d171
aa2a068
ce9d171
91368dd
f68bda2
91368dd
 
 
 
f68bda2
ed1d71f
f68bda2
 
 
 
91368dd
 
 
 
 
 
 
 
 
f68bda2
91368dd
a171703
 
91368dd
fb324e7
91368dd
 
 
 
aa2a068
51bb045
4f1a835
47cd1d4
4f1a835
ed1d71f
 
 
 
4f1a835
 
 
ed1d71f
 
4f1a835
2598099
c00a2ab
ed1d71f
 
91368dd
ce6ce22
 
 
4f1a835
 
 
 
91368dd
ce9d171
 
 
aa2a068
ce9d171
 
 
f68bda2
aa2a068
f68bda2
 
ce9d171
f68bda2
 
 
 
 
755982c
f68bda2
fb324e7
 
 
ed1d71f
 
a078ac1
075c853
 
 
 
a078ac1
 
ed1d71f
7b43476
 
 
 
 
ed1d71f
7b43476
 
ed1d71f
 
50db096
ed1d71f
 
 
 
 
50db096
ed1d71f
 
 
 
 
 
 
20e2e7b
91368dd
ed1d71f
aa2a068
91368dd
95127e7
91368dd
 
 
 
95127e7
91368dd
 
ed1d71f
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
import logging
import queue
from typing import List, NamedTuple
import av
import cv2
import numpy as np
import streamlit as st
from streamlit_webrtc import WebRtcMode, webrtc_streamer
from sample_utils.turn import get_ice_servers
import mediapipe as mp
import os
import time

# Logger Setup
logger = logging.getLogger(_name_)

# Streamlit settings
st.set_page_config(page_title="Virtual Keyboard", page_icon="🏋")
st.title("Interactive Virtual Keyboard")
st.subheader('''Turn on the webcam and use hand gestures to interact with the virtual keyboard.''')

# Initialize MediaPipe and Background Segmentor
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(max_num_hands=1, min_detection_confidence=0.5)
mp_drawing = mp.solutions.drawing_utils

# Virtual Keyboard Layout
keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
        ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
        ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]

class Button:
    def _init_(self, pos, text, size=[100, 100]):
        self.pos = pos
        self.size = size
        self.text = text

class Detection(NamedTuple):
    label: str
    score: float
    box: np.ndarray

result_queue: "queue.Queue[List[Detection]]" = queue.Queue()

indexImg = 0
output_text = ""
prev_key_time = [time.time()] * 2

if "output_text" not in st.session_state:
    st.session_state["output_text"] = ""

# Video Frame Callback with Your Logic
def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
    global indexImg, output_text

    img = frame.to_ndarray(format="bgr24")
    result = hands.process(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))

    # Create the keyboard buttons
    buttonList = []
    h, w = img.shape[:2]
    key_width = int(0.08 * w)  # Increased button width
    key_height = int(0.1 * h)  # Increased button height
    font_scale = 0.005 * w  # Adjusted font size
    font_thickness = int(0.01 * h)  # Adjusted font thickness

    for row, key_row in enumerate(keys):
        for col, key in enumerate(key_row):
            x = int(0.05 * w + col * (key_width + 10))  # Added extra spacing between keys
            y = int(0.05 * h + row * (key_height + 10))  # Added extra spacing between keys
            buttonList.append(Button([x, y], key, size=[key_width, key_height]))

    # Add special buttons for Backspace and Space
    buttonList.append(Button([int(0.85 * w), int(0.05 * h)], 'BS', size=[int(0.1 * w), key_height]))
    buttonList.append(Button([int(0.2 * w), int(0.35 * h)], 'SPACE', size=[int(0.6 * w), key_height]))

    # Draw Keyboard Buttons
    for button in buttonList:
        x, y = button.pos
        bw, bh = button.size
        cv2.rectangle(img, (x, y), (x + bw, y + bh), (200, 200, 200), -1)
        cv2.putText(img, button.text, (x + int(0.2 * bw), y + int(0.7 * bh)), cv2.FONT_HERSHEY_PLAIN, font_scale, (0, 0, 0), font_thickness)

    detections = []
    if result.multi_hand_landmarks:
        for hand_landmarks in result.multi_hand_landmarks:
            mp_drawing.draw_landmarks(
                img, hand_landmarks, mp_hands.HAND_CONNECTIONS,
                mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2, circle_radius=4),
                mp_drawing.DrawingSpec(color=(0, 0, 255), thickness=2)
            )

            h, w, _ = img.shape
            x_min, y_min = w, h
            x_max, y_max = 0, 0
            for lm in hand_landmarks.landmark:
                x, y = int(lm.x * w), int(lm.y * h)
                x_min, y_min = min(x_min, x), min(y_min, y)
                x_max, y_max = max(x_max, x), max(y_max, y)

            bbox = [x_min, y_min, x_max - x_min, y_max - y_min]
            detections.append(Detection(label="Hand", score=0.5, box=np.array(bbox)))

            x4, y4 = int(hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_TIP].x * w), int(hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_TIP].y * h)
            x8, y8 = int(hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].x * w), int(hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y * h)

            distance = np.sqrt((x8 - x4) * 2 + (y8 - y4) * 2)
            click_threshold = 0.2 * np.sqrt(bbox[2] * 2 + bbox[3] * 2)
            
            for button in buttonList:
                x, y = button.pos
                bw, bh = button.size
                if x < x8 < x + bw and y < y8 < y + bh:
                   cv2.rectangle(img, (x, y), (x + bw, y + bh), (0, 255, 160), -1)
                   cv2.putText(img, button.text, (x + int(0.2 * bw), y + int(0.7 * bh)), cv2.FONT_HERSHEY_PLAIN, font_scale, (255, 255, 255), font_thickness)
                   if distance < click_threshold:
                        if time.time() - prev_key_time[0] > 2:
                            prev_key_time[0] = time.time()
                            if button.text != 'BS' and button.text != 'SPACE':
                                output_text += button.text  # Append key to output text 
                            elif button.text == 'BS':
                                output_text = output_text[:-1]  # Remove last character
                            else:
                                output_text += ' '  # Add space

    # Position and dimensions for the output text box
    text_x = int(0.05 * w)
    text_y = int(0.75 * h)
    text_width = int(0.9 * w)
    text_height = int(0.1 * h)

    # Draw the background for output text box
    cv2.rectangle(img, 
              (text_x, text_y - text_height), 
              (text_x + text_width, text_y), 
              (50, 50, 50),  # Dark background for the text area
              -1)

    # Display the output text in white with larger font
    cv2.putText(img, output_text, (text_x + 20, text_y - 20), cv2.FONT_HERSHEY_PLAIN, 2, (255, 255, 255), 5)

    result_queue.put(detections)

    return av.VideoFrame.from_ndarray(img, format="bgr24")

# WebRTC Streamer
webrtc_streamer(
    key="virtual-keyboard",
    mode=WebRtcMode.SENDRECV,
    rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"},
    media_stream_constraints={"video": True, "audio": False},
    video_frame_callback=video_frame_callback,
    async_processing=True,
)