Update app.py

app.py

@@ -8,7 +8,6 @@ import cv2
 import numpy as np
 import streamlit as st
 from streamlit_webrtc import WebRtcMode, webrtc_streamer
-from sample_utils.webrtc_helpers import process_video_frame
 from sample_utils.turn import get_ice_servers
 from cvzone.HandTrackingModule import HandDetector
 from cvzone.SelfiSegmentationModule import SelfiSegmentation
@@ -20,6 +19,63 @@ logger = logging.getLogger(__name__)
 st.title("Interactive Virtual Keyboard with Twilio Integration")
 st.info("Use your webcam to interact with the virtual keyboard via hand gestures.")
 
+class Button:
+    def __init__(self, pos, text, size=[100, 100]):
+        self.pos = pos
+        self.size = size
+        self.text = text
+
+# Function to process the video frame from the webcam
+def process_video_frame(frame: av.VideoFrame, detector, segmentor, imgList, indexImg, keys, session_state)-> av.VideoFrame:
+    # Convert the frame to a numpy array (BGR format)
+    image = frame.to_ndarray(format="bgr24")
+    
+    # Remove background using SelfiSegmentation
+    imgOut = segmentor.removeBG(image, imgList[indexImg])
+
+    # Detect hands on the background-removed image
+    hands, img = detector.findHands(imgOut, flipType=False)
+    
+    # Create a blank canvas for the keyboard
+    keyboard_canvas = np.zeros_like(img)
+    buttonList = []
+
+    # Create buttons for the virtual keyboard based on the keys list
+    for key in keys[0]:
+        buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
+    for key in keys[1]:
+        buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
+    for key in keys[2]:
+        buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))
+
+    # Draw the buttons on the keyboard canvas
+    for button in buttonList:
+        x, y = button.pos
+        cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
+        cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)
+
+    # Handle input and gestures from detected hands
+    if hands:
+        for hand in hands:
+            lmList = hand["lmList"]
+            if lmList:
+                # Get the coordinates of the index finger tip (landmark 8)
+                x8, y8 = lmList[8][0], lmList[8][1]
+                for button in buttonList:
+                    bx, by = button.pos
+                    bw, bh = button.size
+                    # Check if the index finger is over a button
+                    if bx < x8 < bx + bw and by < y8 < by + bh:
+                        # Highlight the button and update the text
+                        cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
+                        cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
+                        # Update the output text in session_state
+                        session_state["output_text"] += button.text
+
+    # Corrected return: Create a video frame from the ndarray image
+    return av.VideoFrame.from_ndarray(img, format="bgr24")
+
+# Initialize components
 detector = HandDetector(maxHands=1, detectionCon=0.8)
 segmentor = SelfiSegmentation()
 keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
@@ -37,17 +93,12 @@ if "output_text" not in st.session_state:
 # Create a thread-safe queue for passing results from callback
 result_queue = queue.Queue()
 
-# Function to process video frame callback
-# Corrected function for video frame callback
-#####FromHERE CHANGE
-# Function to process video frame callback
-# Function to process video frame callback
 def video_frame_callback(frame):
     # Process the frame asynchronously
-    output_text = process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, st.session_state)
-    # Put the processed output text into the queue
-    result_queue.put(output_text)
-    return av.VideoFrame.from_ndarray(flipped, format="bgr24")
+    processed_frame = process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, st.session_state)
+    # Put the processed frame into the queue
+    result_queue.put(processed_frame)
+    return processed_frame
 
 webrtc_ctx = webrtc_streamer(
     key="keyboard-demo",
@@ -67,4 +118,4 @@ st.write(
     1. Turn on your webcam using the checkbox above.
     2. Use hand gestures to interact with the virtual keyboard.
     """
-)
+)