Create webrtc_helpers.py
Browse files
sample_utils/webrtc_helpers.py
ADDED
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import cv2
|
2 |
+
import numpy as np
|
3 |
+
from cvzone.HandTrackingModule import HandDetector
|
4 |
+
import av # Import the av module for video frame handling
|
5 |
+
|
6 |
+
class Button:
|
7 |
+
def __init__(self, pos, text, size=[100, 100]):
|
8 |
+
self.pos = pos
|
9 |
+
self.size = size
|
10 |
+
self.text = text
|
11 |
+
|
12 |
+
# Function to process the video frame from the webcam
|
13 |
+
def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state):
|
14 |
+
# Convert the frame to a numpy array (BGR format)
|
15 |
+
image = frame.to_ndarray(format="bgr24")
|
16 |
+
|
17 |
+
# Remove background using SelfiSegmentation
|
18 |
+
imgOut = segmentor.removeBG(image, imgList[indexImg])
|
19 |
+
|
20 |
+
# Detect hands on the background-removed image
|
21 |
+
hands, img = detector.findHands(imgOut, flipType=False)
|
22 |
+
|
23 |
+
# Create a blank canvas for the keyboard
|
24 |
+
keyboard_canvas = np.zeros_like(img)
|
25 |
+
buttonList = []
|
26 |
+
|
27 |
+
# Create buttons for the virtual keyboard based on the keys list
|
28 |
+
for key in keys[0]:
|
29 |
+
buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
|
30 |
+
for key in keys[1]:
|
31 |
+
buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
|
32 |
+
for key in keys[2]:
|
33 |
+
buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))
|
34 |
+
|
35 |
+
# Draw the buttons on the keyboard canvas
|
36 |
+
for button in buttonList:
|
37 |
+
x, y = button.pos
|
38 |
+
cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
|
39 |
+
cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)
|
40 |
+
|
41 |
+
# Handle input and gestures from detected hands
|
42 |
+
if hands:
|
43 |
+
for hand in hands:
|
44 |
+
lmList = hand["lmList"]
|
45 |
+
if lmList:
|
46 |
+
# Get the coordinates of the index finger tip (landmark 8)
|
47 |
+
x8, y8 = lmList[8][0], lmList[8][1]
|
48 |
+
for button in buttonList:
|
49 |
+
bx, by = button.pos
|
50 |
+
bw, bh = button.size
|
51 |
+
# Check if the index finger is over a button
|
52 |
+
if bx < x8 < bx + bw and by < y8 < by + bh:
|
53 |
+
# Highlight the button and update the text
|
54 |
+
cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
|
55 |
+
cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
|
56 |
+
|
57 |
+
# Update the output text in session_state
|
58 |
+
session_state["output_text"] += button.text
|
59 |
+
|
60 |
+
# Corrected return: Create a video frame from the ndarray image
|
61 |
+
return av.VideoFrame.from_ndarray(img, format="bgr24") # Corrected this line
|