Pratyush101 commited on
Commit
cf222f5
·
verified ·
1 Parent(s): 5b6aa72

Create webrtc_helpers.py

Browse files
Files changed (1) hide show
  1. sample_utils/webrtc_helpers.py +61 -0
sample_utils/webrtc_helpers.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import cv2
2
+ import numpy as np
3
+ from cvzone.HandTrackingModule import HandDetector
4
+ import av # Import the av module for video frame handling
5
+
6
+ class Button:
7
+ def __init__(self, pos, text, size=[100, 100]):
8
+ self.pos = pos
9
+ self.size = size
10
+ self.text = text
11
+
12
+ # Function to process the video frame from the webcam
13
+ def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state):
14
+ # Convert the frame to a numpy array (BGR format)
15
+ image = frame.to_ndarray(format="bgr24")
16
+
17
+ # Remove background using SelfiSegmentation
18
+ imgOut = segmentor.removeBG(image, imgList[indexImg])
19
+
20
+ # Detect hands on the background-removed image
21
+ hands, img = detector.findHands(imgOut, flipType=False)
22
+
23
+ # Create a blank canvas for the keyboard
24
+ keyboard_canvas = np.zeros_like(img)
25
+ buttonList = []
26
+
27
+ # Create buttons for the virtual keyboard based on the keys list
28
+ for key in keys[0]:
29
+ buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
30
+ for key in keys[1]:
31
+ buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
32
+ for key in keys[2]:
33
+ buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))
34
+
35
+ # Draw the buttons on the keyboard canvas
36
+ for button in buttonList:
37
+ x, y = button.pos
38
+ cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
39
+ cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)
40
+
41
+ # Handle input and gestures from detected hands
42
+ if hands:
43
+ for hand in hands:
44
+ lmList = hand["lmList"]
45
+ if lmList:
46
+ # Get the coordinates of the index finger tip (landmark 8)
47
+ x8, y8 = lmList[8][0], lmList[8][1]
48
+ for button in buttonList:
49
+ bx, by = button.pos
50
+ bw, bh = button.size
51
+ # Check if the index finger is over a button
52
+ if bx < x8 < bx + bw and by < y8 < by + bh:
53
+ # Highlight the button and update the text
54
+ cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
55
+ cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
56
+
57
+ # Update the output text in session_state
58
+ session_state["output_text"] += button.text
59
+
60
+ # Corrected return: Create a video frame from the ndarray image
61
+ return av.VideoFrame.from_ndarray(img, format="bgr24") # Corrected this line