Update app.py

app.py

@@ -25,8 +25,20 @@ class Button:
         self.size = size
         self.text = text
 
-# Function to process the video frame from the webcam
-# def process_video_frame(frame: av.VideoFrame, detector, segmentor, imgList, indexImg, keys, session_state)-> av.VideoFrame:
+# # Initialize components
+# detector = HandDetector(maxHands=1, detectionCon=0.8)
+# segmentor = SelfiSegmentation()
+# keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
+#         ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
+#         ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]
+
+# listImg = os.listdir('model/street')
+# imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg]
+# indexImg = 0
+
+
+# # Function to process the video frame from the webcam
+# def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state):
 #     # Convert the frame to a numpy array (BGR format)
 #     image = frame.to_ndarray(format="bgr24")
     
@@ -77,90 +89,21 @@ class Button:
 
 
 
-# Initialize components
-detector = HandDetector(maxHands=1, detectionCon=0.8)
-segmentor = SelfiSegmentation()
-keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
-        ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
-        ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]
-
-listImg = os.listdir('model/street')
-imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg]
-indexImg = 0
-
-
-# Function to process the video frame from the webcam
-def process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, session_state):
-    # Convert the frame to a numpy array (BGR format)
-    image = frame.to_ndarray(format="bgr24")
-    
-    # Remove background using SelfiSegmentation
-    imgOut = segmentor.removeBG(image, imgList[indexImg])
-
-    # Detect hands on the background-removed image
-    hands, img = detector.findHands(imgOut, flipType=False)
-    
-    # Create a blank canvas for the keyboard
-    keyboard_canvas = np.zeros_like(img)
-    buttonList = []
-
-    # Create buttons for the virtual keyboard based on the keys list
-    for key in keys[0]:
-        buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
-    for key in keys[1]:
-        buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
-    for key in keys[2]:
-        buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))
-
-    # Draw the buttons on the keyboard canvas
-    for button in buttonList:
-        x, y = button.pos
-        cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
-        cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)
-
-    # Handle input and gestures from detected hands
-    if hands:
-        for hand in hands:
-            lmList = hand["lmList"]
-            if lmList:
-                # Get the coordinates of the index finger tip (landmark 8)
-                x8, y8 = lmList[8][0], lmList[8][1]
-                for button in buttonList:
-                    bx, by = button.pos
-                    bw, bh = button.size
-                    # Check if the index finger is over a button
-                    if bx < x8 < bx + bw and by < y8 < by + bh:
-                        # Highlight the button and update the text
-                        cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
-                        cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
-                        # Update the output text in session_state
-                        session_state["output_text"] += button.text
-
-    # Corrected return: Create a video frame from the ndarray image
-    return av.VideoFrame.from_ndarray(img, format="bgr24")
-
 
 
 
 
+def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
+    img = frame.to_ndarray(format="bgr24")
+    hands, img = detector.findHands(img, flipType=False)
 
+    # Render hand detection results
+    if hands:
+        hand = hands[0]
+        bbox = hand["bbox"]
+        cv2.rectangle(img, (bbox[0], bbox[1]), (255, 0, 0), 2)
 
-
-
-
-
-
-# def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
-#     img = frame.to_ndarray(format="bgr24")
-#     hands, img = detector.findHands(img, flipType=False)
-
-#     # Render hand detection results
-#     if hands:
-#         hand = hands[0]
-#         bbox = hand["bbox"]
-#         cv2.rectangle(img, (bbox[0], bbox[1]), (255, 0, 0), 2)
-
-#     return av.VideoFrame.from_ndarray(img, format="bgr24")
+    return av.VideoFrame.from_ndarray(img, format="bgr24")
 
 # Shared state for output text
 if "output_text" not in st.session_state:
@@ -169,21 +112,23 @@ if "output_text" not in st.session_state:
 # Create a thread-safe queue for passing results from callback
 result_queue = queue.Queue()
 
-def video_frame_callback(frame):
-    # Process the frame asynchronously
-    processed_frame = process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, st.session_state)
-    # Put the processed frame into the queue
-    result_queue.put(processed_frame)
-    return processed_frame
+# def video_frame_callback(frame):
+#     # Process the frame asynchronously
+#     processed_frame = process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, st.session_state)
+#     # Put the processed frame into the queue
+#     result_queue.put(processed_frame)
+#     return processed_frame
 
 webrtc_ctx = webrtc_streamer(
     key="keyboard-demo",
     mode=WebRtcMode.SENDRECV,
     rtc_configuration={
         "iceServers": get_ice_servers(),
+        "iceTransportPolicy": "relay",
     },
     video_frame_callback=video_frame_callback,
     media_stream_constraints={"video": True, "audio": False},
+    async_processing=True,
 )
 
 st.markdown("### Instructions")