Update websocket_handler.py

websocket_handler.py

@@ -153,7 +153,7 @@ class RealtimeSession:
     def __init__(self, session: Session):
         self.session = session
         self.state = ConversationState.IDLE
-        self.is_websocket_active = True  # Yeni flag
+        self.is_websocket_active = True
         
         # Get settings from config
         config = ConfigProvider.get().global_config.stt_provider.settings
@@ -178,9 +178,15 @@ class RealtimeSession:
         self.audio_chunk_size = config.get("audio_chunk_size", DEFAULT_AUDIO_CHUNK_SIZE)
         self.silence_threshold_ms = silence_threshold
         
+        # Chunk counter için attribute
+        self.chunk_counter = 0
+        
     async def initialize_stt(self):
         """Initialize STT provider"""
         try:
+            # Her başlatmada chunk counter'ı sıfırla
+            self.chunk_counter = 0
+            
             self.stt_manager = STTFactory.create_provider()
             if not self.stt_manager:
                 log_error("❌ STT manager is None - STTFactory.create_provider() returned None", session_id=self.session.session_id)
@@ -192,7 +198,7 @@ class RealtimeSession:
             config = ConfigProvider.get().global_config.stt_provider.settings
             
             # Get language from session locale
-            session_locale = getattr(self.session, 'locale', 'tr')  # Default to 'tr' if not set
+            session_locale = getattr(self.session, 'locale', 'tr')
             
             # Import LocaleManager to get proper locale tag
             from locale_manager import LocaleManager
@@ -225,31 +231,44 @@ class RealtimeSession:
             log_error(f"❌ Failed to initialize STT", error=str(e), traceback=traceback.format_exc(), session_id=self.session.session_id)
             self.stt_manager = None
             self.is_streaming = False
+            self.chunk_counter = 0
             return False
 
     async def restart_stt_if_needed(self):
-        """Restart STT if it's not active - sadece gerektiğinde"""
+        """Restart STT if it's not active"""
         try:
+            # Sadece LISTENING state'inde ve WebSocket aktifse restart yap
             if not self.is_streaming and self.is_websocket_active and self.state == ConversationState.LISTENING:
-                log_info(f"🔄 Restarting STT stream after timeout...", session_id=self.session.session_id)
+                log_info(f"🔄 Restarting STT stream...", session_id=self.session.session_id)
+                
+                # Önce mevcut stream'i temizle
+                await self.stop_stt_streaming()
                 
-                # Mevcut STT manager'ı kullan
-                if self.stt_manager:
-                    # Yeniden başlat
-                    stt_initialized = await self.initialize_stt()
-                    if stt_initialized:
-                        log_info(f"✅ STT stream restarted successfully", session_id=self.session.session_id)
-                        # Reset chunk counter
-                        if hasattr(self, 'chunk_counter'):
-                            self.chunk_counter = 0
-                        return True
-                    else:
-                        log_error(f"❌ Failed to restart STT stream", session_id=self.session.session_id)
-                        return False
+                # Sonra yeniden başlat
+                stt_initialized = await self.initialize_stt()
+                if stt_initialized:
+                    log_info(f"✅ STT stream restarted successfully", session_id=self.session.session_id)
+                    return True
+                else:
+                    log_error(f"❌ Failed to restart STT stream", session_id=self.session.session_id)
+                    return False
             return True
         except Exception as e:
             log_error(f"❌ Error restarting STT", error=str(e), session_id=self.session.session_id)
             return False
+
+    async def stop_stt_streaming(self):
+        """Stop STT streaming completely"""
+        try:
+            if self.stt_manager and self.is_streaming:
+                log_info(f"🛑 Stopping STT stream", session_id=self.session.session_id)
+                await self.stt_manager.stop_streaming()
+                self.is_streaming = False
+                self.chunk_counter = 0
+                log_info(f"✅ STT stream stopped", session_id=self.session.session_id)
+        except Exception as e:
+            log_warning(f"⚠️ Error stopping STT stream: {e}", session_id=self.session.session_id)
+            self.is_streaming = False
     
     async def change_state(self, new_state: ConversationState):
         """Change conversation state"""
@@ -263,23 +282,25 @@ class RealtimeSession:
     
     async def handle_barge_in(self):
         """Handle user interruption"""
-        await self.barge_in_handler.handle_interruption(self.state)
-        await self.change_state(ConversationState.LISTENING)
+        # Barge-in devre dışı - bu metod artık çağrılmamalı
+        log_warning(f"⚠️ Barge-in called but disabled", session_id=self.session.session_id)
+        return
     
     async def reset_for_new_utterance(self):
         """Reset for new user utterance"""
         await self.audio_buffer.clear()
         self.silence_detector.reset()
         self.current_transcription = ""
+        self.chunk_counter = 0  # Chunk counter'ı reset et
         if hasattr(self, 'speech_started'):
-            delattr(self, 'speech_started')  # Speech started flag'ini sıfırla
+            delattr(self, 'speech_started')
+        log_info(f"🔄 Reset for new utterance complete", session_id=self.session.session_id)
     
     async def cleanup(self):
         """Clean up resources"""
         try:
-            self.is_websocket_active = False  # WebSocket kapanıyor
-            if self.stt_manager:
-                await self.stt_manager.stop_streaming()
+            self.is_websocket_active = False
+            await self.stop_stt_streaming()  # STT'yi düzgün durdur
             log_info(f"Cleaned up realtime session", session_id=self.session.session_id)
         except Exception as e:
             log_warning(f"Cleanup error", error=str(e), session_id=self.session.session_id)
@@ -480,7 +501,7 @@ async def websocket_endpoint(websocket: WebSocket, session_id: str):
         
 # ========================= MESSAGE HANDLERS =========================
 async def handle_audio_chunk(websocket: WebSocket, session: RealtimeSession, message: Dict[str, Any]):
-    """Handle incoming audio chunk with barge-in support"""
+    """Handle incoming audio chunk with sequential processing"""
     try:
         # WebSocket kapandıysa işlem yapma
         if not session.is_websocket_active:
@@ -491,14 +512,11 @@ async def handle_audio_chunk(websocket: WebSocket, session: RealtimeSession, mes
             log_warning(f"⚠️ Empty audio chunk received", session_id=session.session.session_id)
             return
         
-        # Check for barge-in during TTS/audio playback
-        if session.state in [ConversationState.PLAYING_AUDIO, ConversationState.PROCESSING_TTS]:
-            await session.handle_barge_in()
-            await websocket.send_json({
-                "type": "control",
-                "action": "stop_playback"
-            })
-            log_info(f"🛑 Barge-in detected", session_id=session.session.session_id, state=session.state.value)
+        # Barge-in devre dışı - TTS/audio playback sırasında audio chunk'ları işleme
+        if session.state in [ConversationState.PLAYING_AUDIO, ConversationState.PROCESSING_TTS, 
+                            ConversationState.PROCESSING_LLM, ConversationState.PROCESSING_STT]:
+            log_debug(f"🔇 Ignoring audio chunk during state: {session.state.value}", session_id=session.session.session_id)
+            return
         
         # Change state to listening if idle
         if session.state == ConversationState.IDLE:
@@ -508,7 +526,14 @@ async def handle_audio_chunk(websocket: WebSocket, session: RealtimeSession, mes
                 "from": "idle",
                 "to": "listening"
             })
+            # IDLE'dan LISTENING'e geçerken STT'yi başlat
+            if not session.is_streaming:
+                await session.restart_stt_if_needed()
         
+        # LISTENING state'inde değilse audio işleme
+        if session.state != ConversationState.LISTENING:
+            return
+            
         # Add to buffer
         await session.audio_buffer.add_chunk(audio_data)
         
@@ -518,14 +543,13 @@ async def handle_audio_chunk(websocket: WebSocket, session: RealtimeSession, mes
         # Check silence
         silence_duration = session.silence_detector.update(decoded_audio)
         
-        # Stream to STT if available
+        # Stream to STT if available and in LISTENING state
         if session.stt_manager and session.state == ConversationState.LISTENING:
             # Ensure streaming is active
             if not session.is_streaming:
-                log_warning(f"⚠️ STT manager exists but streaming not active", session_id=session.session.session_id)
-                # Try to restart streaming
-                stt_initialized = await session.initialize_stt()
-                if not stt_initialized:
+                log_warning(f"⚠️ STT not streaming, attempting to restart", session_id=session.session.session_id)
+                restart_success = await session.restart_stt_if_needed()
+                if not restart_success:
                     await websocket.send_json({
                         "type": "error",
                         "error_type": "stt_error",
@@ -534,9 +558,7 @@ async def handle_audio_chunk(websocket: WebSocket, session: RealtimeSession, mes
                     return
             
             try:
-                # Chunk counter
-                if not hasattr(session, 'chunk_counter'):
-                    session.chunk_counter = 0
+                # Chunk counter artır
                 session.chunk_counter += 1
                 
                 if session.chunk_counter == 1:
@@ -560,9 +582,12 @@ async def handle_audio_chunk(websocket: WebSocket, session: RealtimeSession, mes
                         
                         session.current_transcription = result.text
                         
-                        # Final transcription geldiğinde hemen işle
+                        # Final transcription geldiğinde STT'yi durdur ve işle
                         if session.current_transcription:
-                            # State'i değiştir ve user input'u işle
+                            # Önce STT'yi durdur
+                            await session.stop_stt_streaming()
+                            
+                            # State'i değiştir
                             await session.change_state(ConversationState.PROCESSING_STT)
                             await websocket.send_json({
                                 "type": "state_change",
@@ -576,31 +601,31 @@ async def handle_audio_chunk(websocket: WebSocket, session: RealtimeSession, mes
                             # Reset for new utterance
                             await session.reset_for_new_utterance()
                             return
-                    
-                    # Interim result'ları artık göndermiyoruz ve loglama yapmıyoruz
                         
             except Exception as e:
-                log_error(f"❌ STT streaming error", error=str(e), traceback=traceback.format_exc(), session_id=session.session.session_id)
-                await websocket.send_json({
-                    "type": "error",
-                    "error_type": "stt_error",
-                    "message": f"STT error: {str(e)}"
-                })
+                error_msg = str(e)
+                # Google STT timeout hatası kontrolü
+                if "Audio Timeout Error" in error_msg or "stream duration" in error_msg or "Exceeded maximum allowed stream duration" in error_msg:
+                    log_warning(f"⚠️ STT timeout detected, restarting stream", session_id=session.session.session_id)
+                    session.is_streaming = False
+                    session.chunk_counter = 0
+                    # Timeout durumunda yeniden başlat
+                    await session.restart_stt_if_needed()
+                else:
+                    log_error(f"❌ STT streaming error", error=error_msg, traceback=traceback.format_exc(), session_id=session.session.session_id)
+                    await websocket.send_json({
+                        "type": "error",
+                        "error_type": "stt_error",
+                        "message": f"STT error: {str(e)}"
+                    })
             
     except Exception as e:
-        error_msg = str(e)
-        # Google STT timeout hatası kontrolü
-        if "Audio Timeout Error" in error_msg or "stream duration" in error_msg:
-            log_warning(f"⚠️ STT timeout detected, marking stream as inactive", session_id=session.session.session_id)
-            session.is_streaming = False
-            # Timeout durumunda frontend'e hata gönderme, sessizce handle et
-        else:
-            log_error(f"❌ STT streaming error", error=error_msg, traceback=traceback.format_exc(), session_id=session.session.session_id)
-            await websocket.send_json({
-                "type": "error",
-                "error_type": "stt_error",
-                "message": f"STT error: {str(e)}"
-            })
+        log_error(f"❌ Error in handle_audio_chunk", error=str(e), traceback=traceback.format_exc(), session_id=session.session.session_id)
+        await websocket.send_json({
+            "type": "error",
+            "error_type": "audio_error",
+            "message": f"Audio processing error: {str(e)}"
+        })
         
 async def handle_control_message(websocket: WebSocket, session: RealtimeSession, message: Dict[str, Any]):
     """Handle control messages"""
@@ -617,7 +642,7 @@ async def handle_control_message(websocket: WebSocket, session: RealtimeSession,
             "config": {
                 "silence_threshold_ms": session.silence_threshold_ms,
                 "audio_chunk_size": session.audio_chunk_size,
-                "supports_barge_in": True
+                "supports_barge_in": False  # Barge-in devre dışı
             }
         })
         
@@ -627,16 +652,13 @@ async def handle_control_message(websocket: WebSocket, session: RealtimeSession,
         await websocket.close()
         
     elif action == "interrupt":
-        # Handle explicit interrupt
-        await session.handle_barge_in()
-        await websocket.send_json({
-            "type": "control",
-            "action": "interrupt_acknowledged"
-        })
+        # Barge-in devre dışı - ignore
+        log_warning(f"⚠️ Interrupt request ignored (barge-in disabled)", session_id=session.session.session_id)
         
     elif action == "reset":
         # Reset conversation state
         await session.reset_for_new_utterance()
+        await session.stop_stt_streaming()
         await session.change_state(ConversationState.IDLE)
         await websocket.send_json({
             "type": "state_change",
@@ -647,6 +669,7 @@ async def handle_control_message(websocket: WebSocket, session: RealtimeSession,
     elif action == "audio_ended":
         # Audio playback ended on client
         if session.state == ConversationState.PLAYING_AUDIO:
+            log_info(f"🎵 Client reported audio ended", session_id=session.session.session_id)
             await session.change_state(ConversationState.LISTENING)
             await websocket.send_json({
                 "type": "state_change",
@@ -655,21 +678,19 @@ async def handle_control_message(websocket: WebSocket, session: RealtimeSession,
             })
             # STT'yi yeniden başlat
             await session.restart_stt_if_needed()
-
+            
+    elif action == "restart_stt":
+        # Manual STT restart request
+        log_info(f"🔄 Manual STT restart requested", session_id=session.session.session_id)
+        await session.stop_stt_streaming()
+        await session.restart_stt_if_needed()
 
 # ========================= PROCESSING FUNCTIONS =========================
 async def process_user_input(websocket: WebSocket, session: RealtimeSession):
     """Process complete user input"""
     try:
-        # LLM işlemesi sırasında STT'yi durdur
-        if session.stt_manager and session.is_streaming:
-            log_info(f"⏸️ Pausing STT during LLM processing", session_id=session.session.session_id)
-            try:
-                await session.stt_manager.stop_streaming()
-                session.is_streaming = False
-            except Exception as e:
-                log_warning(f"⚠️ Error stopping STT: {e}", session_id=session.session.session_id)
-                session.is_streaming = False
+        # LLM işlemesi başlamadan önce STT'nin tamamen durduğundan emin ol
+        await session.stop_stt_streaming()
 
         # WebSocket aktif mi kontrol et
         if not session.is_websocket_active:
@@ -677,9 +698,10 @@ async def process_user_input(websocket: WebSocket, session: RealtimeSession):
             
         user_text = session.current_transcription
         if not user_text:
-            log_warning(f"⚠️ Empty transcription, continuing listening", session_id=session.session.session_id)
-            # Boş transcription'da bile listening'de kal
+            log_warning(f"⚠️ Empty transcription, returning to listening", session_id=session.session.session_id)
+            # Boş transcription durumunda listening'e dön ve STT'yi yeniden başlat
             await session.change_state(ConversationState.LISTENING)
+            await session.restart_stt_if_needed()
             return
         
         log_info(f"🎯 Processing user input", text=user_text, session_id=session.session.session_id)
@@ -740,18 +762,25 @@ async def process_user_input(websocket: WebSocket, session: RealtimeSession):
             
             log_info(f"🎵 Starting TTS generation for response", session_id=session.session.session_id)
             
-            # Generate TTS with barge-in support
-            tts_task = await session.barge_in_handler.start_tts_task(
-                generate_and_stream_tts(websocket, session, tts_provider, response_text)
-            )
+            # Generate TTS (barge-in devre dışı)
+            await generate_and_stream_tts(websocket, session, tts_provider, response_text)
+            
+            # TTS bittikten sonra LISTENING state'ine geç
+            await session.change_state(ConversationState.LISTENING)
+            if session.is_websocket_active:
+                await websocket.send_json({
+                    "type": "state_change",
+                    "from": "playing_audio",
+                    "to": "listening"
+                })
+            
+            # STT'yi yeniden başlat
+            log_info(f"🔄 Restarting STT after TTS completion", session_id=session.session.session_id)
+            await session.restart_stt_if_needed()
             
-            try:
-                await tts_task
-            except asyncio.CancelledError:
-                log_info("⚡ TTS cancelled due to barge-in", session_id=session.session.session_id)
         else:
             log_info(f"⚠️ No TTS provider or WebSocket inactive", session_id=session.session.session_id)
-            # No TTS, go back to listening
+            # No TTS, go back to listening and restart STT
             await session.change_state(ConversationState.LISTENING)
             if session.is_websocket_active:
                 await websocket.send_json({
@@ -759,6 +788,7 @@ async def process_user_input(websocket: WebSocket, session: RealtimeSession):
                     "from": "processing_llm",
                     "to": "listening"
                 })
+            await session.restart_stt_if_needed()
         
     except Exception as e:
         log_error(
@@ -773,8 +803,9 @@ async def process_user_input(websocket: WebSocket, session: RealtimeSession):
                 "message": f"Processing error: {str(e)}"
             })
         await session.reset_for_new_utterance()
-        # Hata durumunda bile listening'e dön
+        # Hata durumunda listening'e dön ve STT'yi yeniden başlat
         await session.change_state(ConversationState.LISTENING)
+        await session.restart_stt_if_needed()
 
 async def generate_and_stream_tts(
     websocket: WebSocket,
@@ -782,17 +813,10 @@ async def generate_and_stream_tts(
     tts_provider,
     text: str
 ):
-    """Generate and stream TTS audio with cancellation support"""
+    """Generate and stream TTS audio with sequential processing"""
     try:
-        # TTS başlamadan önce STT'yi durdur - timeout'u önle
-        if session.stt_manager and session.is_streaming:
-            log_info(f"⏸️ Pausing STT stream during TTS", session_id=session.session.session_id)
-            try:
-                await session.stt_manager.stop_streaming()
-                session.is_streaming = False
-            except Exception as e:
-                log_warning(f"⚠️ Error stopping STT before TTS: {e}", session_id=session.session.session_id)
-                session.is_streaming = False
+        # TTS başlamadan önce STT'nin tamamen durduğundan emin ol
+        await session.stop_stt_streaming()
         
         log_info(f"🎤 Starting TTS generation for text: '{text[:50]}...'", session_id=session.session.session_id)
         
@@ -841,11 +865,6 @@ async def generate_and_stream_tts(
         log_info(f"📤 Streaming TTS audio: {len(audio_data)} bytes as {total_length} base64 chars in {total_chunks} chunks", session_id=session.session.session_id)
         
         for i in range(0, total_length, chunk_size):
-            # Check for cancellation
-            if asyncio.current_task().cancelled():
-                log_info(f"⚡ Streaming cancelled at chunk {i//chunk_size}", session_id=session.session.session_id)
-                break
-            
             # WebSocket aktif mi kontrol et
             if not session.is_websocket_active:
                 log_warning(f"⚠️ WebSocket inactive during streaming, stopping", session_id=session.session.session_id)
@@ -877,15 +896,9 @@ async def generate_and_stream_tts(
             audio_size=len(audio_data),
             chunks_sent=total_chunks
         )
-
-        # TTS bitiminde STT'yi yeniden başlat
-        if session.state == ConversationState.LISTENING:
-            log_info(f"🔄 Restarting STT after TTS completion", session_id=session.session.session_id)
-            await session.restart_stt_if_needed()
         
-    except asyncio.CancelledError:
-        log_info("🛑 TTS streaming cancelled", session_id=session.session.session_id)
-        raise
+        # TTS bitimi - state değişimi process_user_input'ta yapılacak
+        
     except Exception as e:
         error_msg = str(e)
         log_error(
@@ -910,11 +923,13 @@ async def generate_and_stream_tts(
                     "message": f"TTS error: {error_msg}"
                 })
         
-        # TTS hatası durumunda idle'a dön
-        await session.change_state(ConversationState.IDLE)
+        # TTS hatası durumunda listening'e dön
+        await session.change_state(ConversationState.LISTENING)
         if session.is_websocket_active:
             await websocket.send_json({
                 "type": "state_change", 
                 "from": "processing_tts",
-                "to": "idle"
-            })
+                "to": "listening"
+            })
+        # STT'yi yeniden başlat
+        await session.restart_stt_if_needed()