app.py

import logging
import queue
from pathlib import Path
from typing import List, NamedTuple

import av
import cv2
import numpy as np
import streamlit as st
from streamlit_webrtc import WebRtcMode, webrtc_streamer
from sample_utils.webrtc_helpers import process_video_frame
from sample_utils.turn import get_ice_servers
from cvzone.HandTrackingModule import HandDetector
from cvzone.SelfiSegmentationModule import SelfiSegmentation
import time
import os

logger = logging.getLogger(__name__)

st.title("Interactive Virtual Keyboard with Twilio Integration")
st.info("Use your webcam to interact with the virtual keyboard via hand gestures.")

detector = HandDetector(maxHands=1, detectionCon=0.8)
segmentor = SelfiSegmentation()
keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"],
        ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";"],
        ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/"]]

listImg = os.listdir('model/street')
imgList = [cv2.imread(f'model/street/{imgPath}') for imgPath in listImg]
indexImg = 0

# Shared state for output text
if "output_text" not in st.session_state:
    st.session_state["output_text"] = ""

# Create a thread-safe queue for passing results from callback
result_queue = queue.Queue()

# Function to process video frame callback
# Corrected function for video frame callback
#####FromHERE CHANGE
# Function to process video frame callback
# Function to process video frame callback
def video_frame_callback(frame):
    # Process the frame asynchronously
    output_text = process_video_frame(frame, detector, segmentor, imgList, indexImg, keys, st.session_state)
    # Put the processed output text into the queue
    result_queue.put(output_text)
    return frame  # Return the processed frame for 

webrtc_ctx = webrtc_streamer(
    key="keyboard-demo",
    mode=WebRtcMode.SENDRECV,
    rtc_configuration={"iceServers": get_ice_servers()},
    video_frame_callback=video_frame_callback,
    media_stream_constraints={"video": True, "audio": False},
)

st.markdown("### Instructions")
st.write(
    """
    1. Turn on your webcam using the checkbox above.
    2. Use hand gestures to interact with the virtual keyboard.
    """
)