app.py

import logging
import queue
from pathlib import Path
from typing import List, NamedTuple

import av
import cv2
import numpy as np
import streamlit as st
from streamlit_webrtc import WebRtcMode, webrtc_streamer
from sample_utils.turn import get_ice_servers
from cvzone.HandTrackingModule import HandDetector
from cvzone.SelfiSegmentationModule import SelfiSegmentation
import time
import os

logger = logging.getLogger(__name__)

st.title("Interactive Virtual Keyboard with Twilio Integration")
st.info("Use your webcam to interact with the virtual keyboard via hand gestures.")

class Button:
    def __init__(self, pos, text, size=[100, 100]):
        self.pos = pos
        self.size = size
        self.text = text

# Function to process the video frame from the webcam
def process_video_frame(frame: av.VideoFrame, detector, segmentor, imgList, indexImg, keys, session_state)-> av.VideoFrame:
    # Convert the frame to a numpy array (BGR format)
    image = frame.to_ndarray(format="bgr24")
    
    # Remove background using SelfiSegmentation
    imgOut = segmentor.removeBG(image, imgList[indexImg])

    # Detect hands on the background-removed image
    hands, img = detector.findHands(imgOut, flipType=False)
    
    # Create a blank canvas for the keyboard
    keyboard_canvas = np.zeros_like(img)
    buttonList = []

    # Create buttons for the virtual keyboard based on the keys list
    for key in keys[0]:
        buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
    for key in keys[1]:
        buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
    for key in keys[2]:
        buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))

    # Draw the buttons on the keyboard canvas
    for button in buttonList:
        x, y = button.pos
        cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
        cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)

    # Handle input and gestures from detected hands
    if hands:
        for hand in hands:
            lmList = hand["lmList"]
            if lmList:
                # Get the coordinates of the index finger tip (landmark 8)
                x8, y8 = lmList[8][0], lmList[8][1]
                for button in buttonList:
                    bx, by = button.pos
                    bw, bh = button.size
                    # Check if the index finger is over a button
                    if bx < x8 < bx + bw and by < y8 < by + bh:
                        # Highlight the button and update the text
                        cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
                        cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
                        # Update the output text in session_state
                        session_state["output_text"] += button.text

    # Corrected return: Create a video frame from the ndarray image
    return av.VideoFrame.from_ndarray(img, format="bgr24")

# Initialize components
detector = HandDetector(maxHands=1, detectionCon=0.8)
segmentor = SelfiSegmentation()
keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
        ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
        ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]

listImg = os.listdir('model/street')
imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg]
indexImg = 0

# Shared state for output text
if "output_text" not in st.session_state:
    st.session_state["output_text"] = ""

# Create a thread-safe queue for passing results from callback
result_queue = queue.Queue()

def video_frame_callback(frame):
    # Process the frame asynchronously
    processed_frame = process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, st.session_state)
    # Put the processed frame into the queue
    result_queue.put(processed_frame)
    return processed_frame

webrtc_ctx = webrtc_streamer(
    key="keyboard-demo",
    mode=WebRtcMode.SENDRECV,
    rtc_configuration={
        "iceServers": get_ice_servers(),
        "iceTransportPolicy": "relay",
    },
    video_frame_callback=video_frame_callback,
    media_stream_constraints={"video": True, "audio": False},
    async_processing=True,
)

st.markdown("### Instructions")
st.write(
    """
    1. Turn on your webcam using the checkbox above.
    2. Use hand gestures to interact with the virtual keyboard.
    """
)