app.py

import logging
import queue
from pathlib import Path
from typing import List, NamedTuple

import av
import cv2
import numpy as np
import streamlit as st
from streamlit_webrtc import WebRtcMode, webrtc_streamer
from sample_utils.turn import get_ice_servers
from cvzone.HandTrackingModule import HandDetector
from cvzone.SelfiSegmentationModule import SelfiSegmentation
import time
import os

logger = logging.getLogger(__name__)

st.title("Interactive Virtual Keyboard with Twilio Integration")
st.info("Use your webcam to interact with the virtual keyboard via hand gestures.")

class Button:
    def __init__(self, pos, text, size=[100, 100]):
        self.pos = pos
        self.size = size
        self.text = text

# Function to process the video frame from the webcam
# def process_video_frame(frame: av.VideoFrame, detector, segmentor, imgList, indexImg, keys, session_state)-> av.VideoFrame:
#     # Convert the frame to a numpy array (BGR format)
#     image = frame.to_ndarray(format="bgr24")
    
#     # Remove background using SelfiSegmentation
#     imgOut = segmentor.removeBG(image, imgList[indexImg])

#     # Detect hands on the background-removed image
#     hands, img = detector.findHands(imgOut, flipType=False)
    
#     # Create a blank canvas for the keyboard
#     keyboard_canvas = np.zeros_like(img)
#     buttonList = []

#     # Create buttons for the virtual keyboard based on the keys list
#     for key in keys[0]:
#         buttonList.append(Button([30 + keys[0].index(key) * 105, 30], key))
#     for key in keys[1]:
#         buttonList.append(Button([30 + keys[1].index(key) * 105, 150], key))
#     for key in keys[2]:
#         buttonList.append(Button([30 + keys[2].index(key) * 105, 260], key))

#     # Draw the buttons on the keyboard canvas
#     for button in buttonList:
#         x, y = button.pos
#         cv2.rectangle(keyboard_canvas, (x, y), (x + button.size[0], y + button.size[1]), (255, 255, 255), -1)
#         cv2.putText(keyboard_canvas, button.text, (x + 20, y + 70), cv2.FONT_HERSHEY_PLAIN, 5, (0, 0, 0), 3)

#     # Handle input and gestures from detected hands
#     if hands:
#         for hand in hands:
#             lmList = hand["lmList"]
#             if lmList:
#                 # Get the coordinates of the index finger tip (landmark 8)
#                 x8, y8 = lmList[8][0], lmList[8][1]
#                 for button in buttonList:
#                     bx, by = button.pos
#                     bw, bh = button.size
#                     # Check if the index finger is over a button
#                     if bx < x8 < bx + bw and by < y8 < by + bh:
#                         # Highlight the button and update the text
#                         cv2.rectangle(img, (bx, by), (bx + bw, by + bh), (0, 255, 0), -1)
#                         cv2.putText(img, button.text, (bx + 20, by + 70), cv2.FONT_HERSHEY_PLAIN, 5, (255, 255, 255), 3)
#                         # Update the output text in session_state
#                         session_state["output_text"] += button.text

#     # Corrected return: Create a video frame from the ndarray image
#     return av.VideoFrame.from_ndarray(img, format="bgr24")


def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
    img = frame.to_ndarray(format="bgr24")
    hands, img = detector.findHands(img, flipType=False)

    # Render hand detection results
    if hands:
        hand = hands[0]
        bbox = hand["bbox"]
        cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[0]+bbox[2], bbox[1]+bbox[3]), (255, 0, 0), 2)

    return av.VideoFrame.from_ndarray(img, format="bgr24")




# Initialize components
detector = HandDetector(maxHands=1, detectionCon=0.8)
segmentor = SelfiSegmentation()
keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
        ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
        ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]

listImg = os.listdir('model/street')
imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg]
indexImg = 0

# Shared state for output text
if "output_text" not in st.session_state:
    st.session_state["output_text"] = ""

# Create a thread-safe queue for passing results from callback
result_queue = queue.Queue()

# def video_frame_callback(frame):
#     # Process the frame asynchronously
#     processed_frame = process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, st.session_state)
#     # Put the processed frame into the queue
#     result_queue.put(processed_frame)
#     return processed_frame

webrtc_ctx = webrtc_streamer(
    key="keyboard-demo",
    mode=WebRtcMode.SENDRECV,
    rtc_configuration={
        "iceServers": get_ice_servers(),
        "iceTransportPolicy": "relay",
    },
    video_frame_callback=video_frame_callback,
    media_stream_constraints={"video": True, "audio": False},
    async_processing=True,
)

st.markdown("### Instructions")
st.write(
    """
    1. Turn on your webcam using the checkbox above.
    2. Use hand gestures to interact with the virtual keyboard.
    """
)