Pratyush101 commited on
Commit
a472ccb
·
verified ·
1 Parent(s): 89f5596

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +308 -308
app.py CHANGED
@@ -1,382 +1,382 @@
1
- # import logging
2
- # import queue
3
- # from pathlib import Path
4
- # from typing import List, NamedTuple
5
- # import mediapipe as mp
6
- # import av
7
- # import cv2
8
- # import numpy as np
9
- # import streamlit as st
10
- # from streamlit_webrtc import WebRtcMode, webrtc_streamer
11
- # from sample_utils.turn import get_ice_servers
12
- # from cvzone.HandTrackingModule import HandDetector
13
- # from cvzone.SelfiSegmentationModule import SelfiSegmentation
14
- # import time
15
- # import os
16
 
17
- # logger = logging.getLogger(__name__)
18
 
19
- # st.title("Interactive Virtual Keyboard with Twilio Integration")
20
- # st.info("Use your webcam to interact with the virtual keyboard via hand gestures.")
21
 
22
- # class Button:
23
- # def __init__(self, pos, text, size=[100, 100]):
24
- # self.pos = pos
25
- # self.size = size
26
- # self.text = text
27
 
28
- # # Initialize components
29
- # detector = HandDetector(maxHands=1, detectionCon=0.8)
30
- # # segmentor = SelfiSegmentation()
31
- # # keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
32
- # # ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
33
- # # ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]
34
 
35
- # # listImg = os.listdir('model/street')
36
- # # imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg]
37
- # # indexImg = 0
38
 
39
 
40
- # # # Function to process the video frame from the webcam
41
- # # def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state):
42
- # # # Convert the frame to a numpy array (BGR format)
43
- # # image = frame.to_ndarray(format="bgr24")
44
 
45
- # # # Remove background using SelfiSegmentation
46
- # # imgOut = segmentor.removeBG(image, imgList[indexImg])
47
 
48
- # # # Detect hands on the background-removed image
49
- # # hands, img = detector.findHands(imgOut, flipType=False)
50
 
51
- # # # Create a blank canvas for the keyboard
52
- # # keyboard_canvas = np.zeros_like(img)
53
- # # buttonList = []
54
-
55
- # # # Create buttons for the virtual keyboard based on the keys list
56
- # # for key in keys[0]:
57
- # # buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
58
- # # for key in keys[1]:
59
- # # buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
60
- # # for key in keys[2]:
61
- # # buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))
62
-
63
- # # # Draw the buttons on the keyboard canvas
64
- # # for button in buttonList:
65
- # # x, y = button.pos
66
- # # cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
67
- # # cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)
68
-
69
- # # # Handle input and gestures from detected hands
70
- # # if hands:
71
- # # for hand in hands:
72
- # # lmList = hand["lmList"]
73
- # # if lmList:
74
- # # # Get the coordinates of the index finger tip (landmark 8)
75
- # # x8, y8 = lmList[8][0], lmList[8][1]
76
- # # for button in buttonList:
77
- # # bx, by = button.pos
78
- # # bw, bh = button.size
79
- # # # Check if the index finger is over a button
80
- # # if bx < x8 < bx + bw and by < y8 < by + bh:
81
- # # # Highlight the button and update the text
82
- # # cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
83
- # # cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
84
- # # # Update the output text in session_state
85
- # # session_state["output_text"] += button.text
86
-
87
- # # # Corrected return: Create a video frame from the ndarray image
88
- # # return av.VideoFrame.from_ndarray(img, format="bgr24")
89
 
90
 
91
 
92
 
93
 
94
 
95
- # # Shared state for output text
96
- # if "output_text" not in st.session_state:
97
- # st.session_state["output_text"] = ""
98
 
99
- # class Detection(NamedTuple):
100
- # label: str
101
- # score: float
102
- # box: np.ndarray
103
 
104
 
105
- # @st.cache_resource # Cache label colors
106
- # def generate_label_colors():
107
- # return np.random.uniform(0, 255, size=(2, 3)) # Two classes: Left and Right Hand
108
 
109
 
110
- # COLORS = generate_label_colors()
111
 
112
- # # Initialize MediaPipe Hands
113
- # mp_hands = mp.solutions.hands
114
- # detector = mp_hands.Hands(static_image_mode=False, max_num_hands=2, min_detection_confidence=0.5)
115
 
116
- # # Session-specific caching
117
- # result_queue: "queue.Queue[List[Detection]]" = queue.Queue()
118
 
119
- # # Hand detection callback
120
- # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
121
- # image = frame.to_ndarray(format="bgr24")
122
- # h, w = image.shape[:2]
123
 
124
- # # Process image with MediaPipe Hands
125
- # results = detector.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
126
-
127
- # detections = []
128
- # if results.multi_hand_landmarks:
129
- # for hand_landmarks, hand_class in zip(results.multi_hand_landmarks, results.multi_handedness):
130
- # # Extract bounding box
131
- # x_min, y_min = 1, 1
132
- # x_max, y_max = 0, 0
133
- # for lm in hand_landmarks.landmark:
134
- # x_min = min(x_min, lm.x)
135
- # y_min = min(y_min, lm.y)
136
- # x_max = max(x_max, lm.x)
137
- # y_max = max(y_max, lm.y)
138
-
139
- # # Scale bbox to image size
140
- # box = np.array([x_min * w, y_min * h, x_max * w, y_max * h]).astype("int")
141
-
142
- # # Label and score
143
- # label = hand_class.classification[0].label
144
- # score = hand_class.classification[0].score
145
-
146
- # detections.append(Detection(label=label, score=score, box=box))
147
-
148
- # # Draw bounding box and label
149
- # color = COLORS[0 if label == "Left" else 1]
150
- # cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), color, 2)
151
- # caption = f"{label}: {round(score * 100, 2)}%"
152
- # cv2.putText(
153
- # image,
154
- # caption,
155
- # (box[0], box[1] - 15 if box[1] - 15 > 15 else box[1] + 15),
156
- # cv2.FONT_HERSHEY_SIMPLEX,
157
- # 0.5,
158
- # color,
159
- # 2,
160
- # )
161
-
162
- # # Put results in the queue
163
- # result_queue.put(detections)
164
 
165
- # return av.VideoFrame.from_ndarray(image, format="bgr24")
166
 
167
 
168
 
169
- # webrtc_ctx = webrtc_streamer(
170
- # key="keyboard-demo",
171
- # mode=WebRtcMode.SENDRECV,
172
- # rtc_configuration={
173
- # "iceServers": get_ice_servers(),
174
- # "iceTransportPolicy": "relay",
175
- # },
176
- # video_frame_callback=video_frame_callback,
177
- # media_stream_constraints={"video": True, "audio": False},
178
- # async_processing=True,
179
- # )
180
 
181
 
182
- # st.markdown("### Instructions")
183
- # st.write(
184
- # """
185
- # 1. Turn on your webcam using the checkbox above.
186
- # 2. Use hand gestures to interact with the virtual keyboard.
187
- # """
188
- # )
189
 
190
 
191
 
192
 
193
- import logging
194
- import cv2
195
- import numpy as np
196
- import streamlit as st
197
- from streamlit_webrtc import WebRtcMode, webrtc_streamer
198
- from cvzone.HandTrackingModule import HandDetector
199
- from cvzone.SelfiSegmentationModule import SelfiSegmentation
200
- import os
201
- import time
202
- import av
203
- import queue
204
- from typing import List, NamedTuple
205
- from sample_utils.turn import get_ice_servers
206
 
207
- logger = logging.getLogger(__name__)
208
 
209
- # Streamlit settings
210
- st.set_page_config(page_title="Virtual Keyboard", layout="wide")
211
- st.title("Interactive Virtual Keyboard")
212
- st.subheader('''Turn on the webcam and use hand gestures to interact with the virtual keyboard.
213
- Use 'a' and 'd' from the keyboard to change the background.''')
214
 
215
- # Initialize modules
216
- detector = HandDetector(maxHands=1, detectionCon=0.8)
217
- segmentor = SelfiSegmentation()
218
 
219
- # Define virtual keyboard layout
220
- keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
221
- ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
222
- ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]
223
 
224
- class Button:
225
- def __init__(self, pos, text, size=[100, 100]):
226
- self.pos = pos
227
- self.size = size
228
- self.text = text
229
 
230
- class Detection(NamedTuple):
231
- label: str
232
- score: float
233
- box: np.ndarray
234
 
235
- # result_queue: "queue.Queue[List[Detection]]" = queue.Queue()
236
 
237
- listImg = os.listdir('model/street') if os.path.exists('model/street') else []
238
- if not listImg:
239
- st.error("Error: 'street' directory is missing or empty. Please add background images.")
240
- st.stop()
241
- else:
242
- imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg if cv2.imread(f'model/street/{imgPath}') is not None]
243
 
244
- indexImg = 0
245
- prev_key_time = [time.time()] * 2
246
- output_text = ""
247
 
248
- if "output_text" not in st.session_state:
249
- st.session_state["output_text"] = ""
250
 
251
 
252
- # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
253
- # img = frame.to_ndarray(format="bgr24")
254
- # hands, img = detector.findHands(img, flipType=False)
255
 
256
- # # Render hand detection results
257
 
258
- # if hands:
259
- # hand = hands[0]
260
- # bbox = hand["bbox"]
261
- # cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[0]+bbox[2], bbox[1]+bbox[3]), (255, 0, 0), 2)
262
 
263
- # cv2.putText(img, 'OpenCV', (50,50), font,
264
- # fontScale, color, thickness, cv2.LINE_AA)
265
- # cv2.putText(img, 'OpenCV', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 1, cv2.LINE_AA)
266
 
267
- # result_queue.put(hands)
268
 
269
- # return av.VideoFrame.from_ndarray(img, format="bgr24")
270
-
271
 
272
- result_queue: "queue.Queue[List[Detection]]" = queue.Queue()
273
 
 
274
 
275
- def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
276
- image = frame.to_ndarray(format="bgr24")
277
 
278
- # Run inference
279
- blob = cv2.dnn.blobFromImage(
280
- cv2.resize(image, (300, 300)), 0.007843, (300, 300), 127.5
281
- )
282
- net.setInput(blob)
283
- output = net.forward()
284
 
285
- h, w = image.shape[:2]
 
 
 
 
 
286
 
287
- # Convert the output array into a structured form.
288
- output = output.squeeze() # (1, 1, N, 7) -> (N, 7)
289
- output = output[output[:, 2] >= score_threshold]
290
- detections = [
291
- Detection(
292
- class_id=int(detection[1]),
293
- label=CLASSES[int(detection[1])],
294
- score=float(detection[2]),
295
- box=(detection[3:7] * np.array([w, h, w, h])),
296
- )
297
- for detection in output
298
- ]
299
-
300
- # Render bounding boxes and captions
301
- for detection in detections:
302
- caption = f"{detection.label}: {round(detection.score * 100, 2)}%"
303
- color = COLORS[detection.class_id]
304
- xmin, ymin, xmax, ymax = detection.box.astype("int")
305
-
306
- cv2.rectangle(image, (xmin, ymin), (xmax, ymax), color, 2)
307
- cv2.putText(
308
- image,
309
- caption,
310
- (xmin, ymin - 15 if ymin - 15 > 15 else ymin + 15),
311
- cv2.FONT_HERSHEY_SIMPLEX,
312
- 0.5,
313
- color,
314
- 2,
315
- )
316
 
317
- result_queue.put(detections)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
318
 
319
- return av.VideoFrame.from_ndarray(image, format="bgr24")
320
 
 
321
 
322
- # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
323
- # global indexImg, output_text
324
 
325
- # img = frame.to_ndarray(format="bgr24")
326
- # imgOut = segmentor.removeBG(img, imgList[indexImg])
327
- # hands, imgOut = detector.findHands(imgOut, flipType=False)
328
 
329
- # buttonList = [Button([30 + col * 105, 30 + row * 120], key) for row, line in enumerate(keys) for col, key in enumerate(line)]
 
 
330
 
331
- # detections = []
332
- # if hands:
333
- # for i, hand in enumerate(hands):
334
- # lmList = hand['lmList']
335
- # bbox = hand['bbox']
336
- # label = "Hand"
337
- # score = hand['score']
338
- # box = np.array([bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]])
339
- # detections.append(Detection(label=label, score=score, box=box))
340
 
341
- # if lmList:
342
- # x4, y4 = lmList[4][0], lmList[4][1]
343
- # x8, y8 = lmList[8][0], lmList[8][1]
344
- # distance = np.sqrt((x8 - x4) ** 2 + (y8 - y4) ** 2)
345
- # click_threshold = 10
 
 
 
 
346
 
347
- # for button in buttonList:
348
- # x, y = button.pos
349
- # w, h = button.size
350
- # if x < x8 < x + w and y < y8 < y + h:
351
- # cv2.rectangle(imgOut, button.pos, (x + w, y + h), (0, 255, 160), -1)
352
- # cv2.putText(imgOut, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
353
-
354
- # if (distance / np.sqrt(bbox[2] ** 2 + bbox[3] ** 2)) * 100 < click_threshold:
355
- # if time.time() - prev_key_time[i] > 2:
356
- # prev_key_time[i] = time.time()
357
- # if button.text != 'BS' and button.text != 'SPACE':
358
- # output_text += button.text
359
- # elif button.text == 'BS':
360
- # output_text = output_text[:-1]
361
- # else:
362
- # output_text += ' '
363
 
364
- # result_queue.put(detections)
365
- # st.session_state["output_text"] = output_text
366
- # return av.VideoFrame.from_ndarray(imgOut, format="bgr24")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
367
 
368
 
369
 
370
- webrtc_streamer(
371
- key="virtual-keyboard",
372
- mode=WebRtcMode.SENDRECV,
373
- rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"},
374
- media_stream_constraints={"video": True, "audio": False},
375
- video_frame_callback=video_frame_callback,
376
- async_processing=True,
377
- )
378
 
379
- st.subheader("Output Text")
380
- st.text_area("Live Input:", value=st.session_state["output_text"], height=200)
381
 
382
 
 
1
+ import logging
2
+ import queue
3
+ from pathlib import Path
4
+ from typing import List, NamedTuple
5
+ import mediapipe as mp
6
+ import av
7
+ import cv2
8
+ import numpy as np
9
+ import streamlit as st
10
+ from streamlit_webrtc import WebRtcMode, webrtc_streamer
11
+ from sample_utils.turn import get_ice_servers
12
+ from cvzone.HandTrackingModule import HandDetector
13
+ from cvzone.SelfiSegmentationModule import SelfiSegmentation
14
+ import time
15
+ import os
16
 
17
+ logger = logging.getLogger(__name__)
18
 
19
+ st.title("Interactive Virtual Keyboard with Twilio Integration")
20
+ st.info("Use your webcam to interact with the virtual keyboard via hand gestures.")
21
 
22
+ class Button:
23
+ def __init__(self, pos, text, size=[100, 100]):
24
+ self.pos = pos
25
+ self.size = size
26
+ self.text = text
27
 
28
+ # Initialize components
29
+ detector = HandDetector(maxHands=1, detectionCon=0.8)
30
+ # segmentor = SelfiSegmentation()
31
+ # keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
32
+ # ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
33
+ # ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]
34
 
35
+ # listImg = os.listdir('model/street')
36
+ # imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg]
37
+ # indexImg = 0
38
 
39
 
40
+ # # Function to process the video frame from the webcam
41
+ # def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state):
42
+ # # Convert the frame to a numpy array (BGR format)
43
+ # image = frame.to_ndarray(format="bgr24")
44
 
45
+ # # Remove background using SelfiSegmentation
46
+ # imgOut = segmentor.removeBG(image, imgList[indexImg])
47
 
48
+ # # Detect hands on the background-removed image
49
+ # hands, img = detector.findHands(imgOut, flipType=False)
50
 
51
+ # # Create a blank canvas for the keyboard
52
+ # keyboard_canvas = np.zeros_like(img)
53
+ # buttonList = []
54
+
55
+ # # Create buttons for the virtual keyboard based on the keys list
56
+ # for key in keys[0]:
57
+ # buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
58
+ # for key in keys[1]:
59
+ # buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
60
+ # for key in keys[2]:
61
+ # buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))
62
+
63
+ # # Draw the buttons on the keyboard canvas
64
+ # for button in buttonList:
65
+ # x, y = button.pos
66
+ # cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
67
+ # cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)
68
+
69
+ # # Handle input and gestures from detected hands
70
+ # if hands:
71
+ # for hand in hands:
72
+ # lmList = hand["lmList"]
73
+ # if lmList:
74
+ # # Get the coordinates of the index finger tip (landmark 8)
75
+ # x8, y8 = lmList[8][0], lmList[8][1]
76
+ # for button in buttonList:
77
+ # bx, by = button.pos
78
+ # bw, bh = button.size
79
+ # # Check if the index finger is over a button
80
+ # if bx < x8 < bx + bw and by < y8 < by + bh:
81
+ # # Highlight the button and update the text
82
+ # cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
83
+ # cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
84
+ # # Update the output text in session_state
85
+ # session_state["output_text"] += button.text
86
+
87
+ # # Corrected return: Create a video frame from the ndarray image
88
+ # return av.VideoFrame.from_ndarray(img, format="bgr24")
89
 
90
 
91
 
92
 
93
 
94
 
95
+ # Shared state for output text
96
+ if "output_text" not in st.session_state:
97
+ st.session_state["output_text"] = ""
98
 
99
+ class Detection(NamedTuple):
100
+ label: str
101
+ score: float
102
+ box: np.ndarray
103
 
104
 
105
+ @st.cache_resource # Cache label colors
106
+ def generate_label_colors():
107
+ return np.random.uniform(0, 255, size=(2, 3)) # Two classes: Left and Right Hand
108
 
109
 
110
+ COLORS = generate_label_colors()
111
 
112
+ # Initialize MediaPipe Hands
113
+ mp_hands = mp.solutions.hands
114
+ detector = mp_hands.Hands(static_image_mode=False, max_num_hands=2, min_detection_confidence=0.5)
115
 
116
+ # Session-specific caching
117
+ result_queue: "queue.Queue[List[Detection]]" = queue.Queue()
118
 
119
+ # Hand detection callback
120
+ def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
121
+ image = frame.to_ndarray(format="bgr24")
122
+ h, w = image.shape[:2]
123
 
124
+ # Process image with MediaPipe Hands
125
+ results = detector.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
126
+
127
+ detections = []
128
+ if results.multi_hand_landmarks:
129
+ for hand_landmarks, hand_class in zip(results.multi_hand_landmarks, results.multi_handedness):
130
+ # Extract bounding box
131
+ x_min, y_min = 1, 1
132
+ x_max, y_max = 0, 0
133
+ for lm in hand_landmarks.landmark:
134
+ x_min = min(x_min, lm.x)
135
+ y_min = min(y_min, lm.y)
136
+ x_max = max(x_max, lm.x)
137
+ y_max = max(y_max, lm.y)
138
+
139
+ # Scale bbox to image size
140
+ box = np.array([x_min * w, y_min * h, x_max * w, y_max * h]).astype("int")
141
+
142
+ # Label and score
143
+ label = hand_class.classification[0].label
144
+ score = hand_class.classification[0].score
145
+
146
+ detections.append(Detection(label=label, score=score, box=box))
147
+
148
+ # Draw bounding box and label
149
+ color = COLORS[0 if label == "Left" else 1]
150
+ cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), color, 2)
151
+ caption = f"{label}: {round(score * 100, 2)}%"
152
+ cv2.putText(
153
+ image,
154
+ caption,
155
+ (box[0], box[1] - 15 if box[1] - 15 > 15 else box[1] + 15),
156
+ cv2.FONT_HERSHEY_SIMPLEX,
157
+ 0.5,
158
+ color,
159
+ 2,
160
+ )
161
+
162
+ # Put results in the queue
163
+ result_queue.put(detections)
164
 
165
+ return av.VideoFrame.from_ndarray(image, format="bgr24")
166
 
167
 
168
 
169
+ webrtc_ctx = webrtc_streamer(
170
+ key="keyboard-demo",
171
+ mode=WebRtcMode.SENDRECV,
172
+ rtc_configuration={
173
+ "iceServers": get_ice_servers(),
174
+ "iceTransportPolicy": "relay",
175
+ },
176
+ video_frame_callback=video_frame_callback,
177
+ media_stream_constraints={"video": True, "audio": False},
178
+ async_processing=True,
179
+ )
180
 
181
 
182
+ st.markdown("### Instructions")
183
+ st.write(
184
+ """
185
+ 1. Turn on your webcam using the checkbox above.
186
+ 2. Use hand gestures to interact with the virtual keyboard.
187
+ """
188
+ )
189
 
190
 
191
 
192
 
193
+ # import logging
194
+ # import cv2
195
+ # import numpy as np
196
+ # import streamlit as st
197
+ # from streamlit_webrtc import WebRtcMode, webrtc_streamer
198
+ # from cvzone.HandTrackingModule import HandDetector
199
+ # from cvzone.SelfiSegmentationModule import SelfiSegmentation
200
+ # import os
201
+ # import time
202
+ # import av
203
+ # import queue
204
+ # from typing import List, NamedTuple
205
+ # from sample_utils.turn import get_ice_servers
206
 
207
+ # logger = logging.getLogger(__name__)
208
 
209
+ # # Streamlit settings
210
+ # st.set_page_config(page_title="Virtual Keyboard", layout="wide")
211
+ # st.title("Interactive Virtual Keyboard")
212
+ # st.subheader('''Turn on the webcam and use hand gestures to interact with the virtual keyboard.
213
+ # Use 'a' and 'd' from the keyboard to change the background.''')
214
 
215
+ # # Initialize modules
216
+ # detector = HandDetector(maxHands=1, detectionCon=0.8)
217
+ # segmentor = SelfiSegmentation()
218
 
219
+ # # Define virtual keyboard layout
220
+ # keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
221
+ # ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
222
+ # ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]
223
 
224
+ # class Button:
225
+ # def __init__(self, pos, text, size=[100, 100]):
226
+ # self.pos = pos
227
+ # self.size = size
228
+ # self.text = text
229
 
230
+ # class Detection(NamedTuple):
231
+ # label: str
232
+ # score: float
233
+ # box: np.ndarray
234
 
235
+ # # result_queue: "queue.Queue[List[Detection]]" = queue.Queue()
236
 
237
+ # listImg = os.listdir('model/street') if os.path.exists('model/street') else []
238
+ # if not listImg:
239
+ # st.error("Error: 'street' directory is missing or empty. Please add background images.")
240
+ # st.stop()
241
+ # else:
242
+ # imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg if cv2.imread(f'model/street/{imgPath}') is not None]
243
 
244
+ # indexImg = 0
245
+ # prev_key_time = [time.time()] * 2
246
+ # output_text = ""
247
 
248
+ # if "output_text" not in st.session_state:
249
+ # st.session_state["output_text"] = ""
250
 
251
 
252
+ # # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
253
+ # # img = frame.to_ndarray(format="bgr24")
254
+ # # hands, img = detector.findHands(img, flipType=False)
255
 
256
+ # # # Render hand detection results
257
 
258
+ # # if hands:
259
+ # # hand = hands[0]
260
+ # # bbox = hand["bbox"]
261
+ # # cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[0]+bbox[2], bbox[1]+bbox[3]), (255, 0, 0), 2)
262
 
263
+ # # cv2.putText(img, 'OpenCV', (50,50), font,
264
+ # # fontScale, color, thickness, cv2.LINE_AA)
265
+ # # cv2.putText(img, 'OpenCV', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 1, cv2.LINE_AA)
266
 
267
+ # # result_queue.put(hands)
268
 
269
+ # # return av.VideoFrame.from_ndarray(img, format="bgr24")
 
270
 
 
271
 
272
+ # result_queue: "queue.Queue[List[Detection]]" = queue.Queue()
273
 
 
 
274
 
275
+ # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
276
+ # image = frame.to_ndarray(format="bgr24")
 
 
 
 
277
 
278
+ # # Run inference
279
+ # blob = cv2.dnn.blobFromImage(
280
+ # cv2.resize(image, (300, 300)), 0.007843, (300, 300), 127.5
281
+ # )
282
+ # net.setInput(blob)
283
+ # output = net.forward()
284
 
285
+ # h, w = image.shape[:2]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
286
 
287
+ # # Convert the output array into a structured form.
288
+ # output = output.squeeze() # (1, 1, N, 7) -> (N, 7)
289
+ # output = output[output[:, 2] >= score_threshold]
290
+ # detections = [
291
+ # Detection(
292
+ # class_id=int(detection[1]),
293
+ # label=CLASSES[int(detection[1])],
294
+ # score=float(detection[2]),
295
+ # box=(detection[3:7] * np.array([w, h, w, h])),
296
+ # )
297
+ # for detection in output
298
+ # ]
299
+
300
+ # # Render bounding boxes and captions
301
+ # for detection in detections:
302
+ # caption = f"{detection.label}: {round(detection.score * 100, 2)}%"
303
+ # color = COLORS[detection.class_id]
304
+ # xmin, ymin, xmax, ymax = detection.box.astype("int")
305
+
306
+ # cv2.rectangle(image, (xmin, ymin), (xmax, ymax), color, 2)
307
+ # cv2.putText(
308
+ # image,
309
+ # caption,
310
+ # (xmin, ymin - 15 if ymin - 15 > 15 else ymin + 15),
311
+ # cv2.FONT_HERSHEY_SIMPLEX,
312
+ # 0.5,
313
+ # color,
314
+ # 2,
315
+ # )
316
 
317
+ # result_queue.put(detections)
318
 
319
+ # return av.VideoFrame.from_ndarray(image, format="bgr24")
320
 
 
 
321
 
322
+ # # def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
323
+ # # global indexImg, output_text
 
324
 
325
+ # # img = frame.to_ndarray(format="bgr24")
326
+ # # imgOut = segmentor.removeBG(img, imgList[indexImg])
327
+ # # hands, imgOut = detector.findHands(imgOut, flipType=False)
328
 
329
+ # # buttonList = [Button([30 + col * 105, 30 + row * 120], key) for row, line in enumerate(keys) for col, key in enumerate(line)]
 
 
 
 
 
 
 
 
330
 
331
+ # # detections = []
332
+ # # if hands:
333
+ # # for i, hand in enumerate(hands):
334
+ # # lmList = hand['lmList']
335
+ # # bbox = hand['bbox']
336
+ # # label = "Hand"
337
+ # # score = hand['score']
338
+ # # box = np.array([bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]])
339
+ # # detections.append(Detection(label=label, score=score, box=box))
340
 
341
+ # # if lmList:
342
+ # # x4, y4 = lmList[4][0], lmList[4][1]
343
+ # # x8, y8 = lmList[8][0], lmList[8][1]
344
+ # # distance = np.sqrt((x8 - x4) ** 2 + (y8 - y4) ** 2)
345
+ # # click_threshold = 10
 
 
 
 
 
 
 
 
 
 
 
346
 
347
+ # # for button in buttonList:
348
+ # # x, y = button.pos
349
+ # # w, h = button.size
350
+ # # if x < x8 < x + w and y < y8 < y + h:
351
+ # # cv2.rectangle(imgOut, button.pos, (x + w, y + h), (0, 255, 160), -1)
352
+ # # cv2.putText(imgOut, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
353
+
354
+ # # if (distance / np.sqrt(bbox[2] ** 2 + bbox[3] ** 2)) * 100 < click_threshold:
355
+ # # if time.time() - prev_key_time[i] > 2:
356
+ # # prev_key_time[i] = time.time()
357
+ # # if button.text != 'BS' and button.text != 'SPACE':
358
+ # # output_text += button.text
359
+ # # elif button.text == 'BS':
360
+ # # output_text = output_text[:-1]
361
+ # # else:
362
+ # # output_text += ' '
363
+
364
+ # # result_queue.put(detections)
365
+ # # st.session_state["output_text"] = output_text
366
+ # # return av.VideoFrame.from_ndarray(imgOut, format="bgr24")
367
 
368
 
369
 
370
+ # webrtc_streamer(
371
+ # key="virtual-keyboard",
372
+ # mode=WebRtcMode.SENDRECV,
373
+ # rtc_configuration={"iceServers": get_ice_servers(), "iceTransportPolicy": "relay"},
374
+ # media_stream_constraints={"video": True, "audio": False},
375
+ # video_frame_callback=video_frame_callback,
376
+ # async_processing=True,
377
+ # )
378
 
379
+ # st.subheader("Output Text")
380
+ # st.text_area("Live Input:", value=st.session_state["output_text"], height=200)
381
 
382