Add application file

Dockerfile

@@ -0,0 +1,44 @@
+# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.10-slim
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    curl \
+    ffmpeg \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Set working directory
+WORKDIR /app
+
+# Create static directory and set permissions
+RUN mkdir -p /app/static && chmod 777 /app/static
+
+# Create a non-root user
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+# Copy requirements first to leverage Docker cache
+COPY --chown=user requirements.txt .
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Copy application files
+COPY --chown=user *.py /app/
+COPY --chown=user whisper_streaming_custom /app/whisper_streaming_custom/
+COPY --chown=user diarization /app/diarization/
+COPY --chown=user static /app/static/
+
+# Set environment variables
+ENV PYTHONPATH=/app
+ENV PYTHONUNBUFFERED=1
+
+# Expose the port the server runs on
+EXPOSE 7860
+
+# Run the server using main.py
+CMD ["python", "main.py", "--host", "0.0.0.0", "--port", "7860", "--model", "tiny", "--backend", "faster-whisper", "--task", "transcribe"]

__init__.py

@@ -0,0 +1,4 @@
+from .core import WhisperLiveKit, parse_args
+from .audio_processor import AudioProcessor
+
+__all__ = ['WhisperLiveKit', 'AudioProcessor', 'parse_args']

audio_processor.py

@@ -0,0 +1,409 @@
+import asyncio
+import numpy as np
+import ffmpeg
+from time import time, sleep
+import math
+import logging
+import traceback
+from datetime import timedelta
+from typing import List, Dict, Any
+from timed_objects import ASRToken
+from whisper_streaming_custom.whisper_online import online_factory
+from core import WhisperLiveKit
+
+# Set up logging once
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+def format_time(seconds: float) -> str:
+    """Format seconds as HH:MM:SS."""
+    return str(timedelta(seconds=int(seconds)))
+
+class AudioProcessor:
+    """
+    Processes audio streams for transcription and diarization.
+    Handles audio processing, state management, and result formatting.
+    """
+    
+    def __init__(self):
+        """Initialize the audio processor with configuration, models, and state."""
+        
+        models = WhisperLiveKit()
+        
+        # Audio processing settings
+        self.args = models.args
+        self.sample_rate = 16000
+        self.channels = 1
+        self.samples_per_sec = int(self.sample_rate * self.args.min_chunk_size)
+        self.bytes_per_sample = 2
+        self.bytes_per_sec = self.samples_per_sec * self.bytes_per_sample
+        self.max_bytes_per_sec = 32000 * 5  # 5 seconds of audio at 32 kHz
+        
+        # State management
+        self.tokens = []
+        self.buffer_transcription = ""
+        self.buffer_diarization = ""
+        self.full_transcription = ""
+        self.end_buffer = 0
+        self.end_attributed_speaker = 0
+        self.lock = asyncio.Lock()
+        self.beg_loop = time()
+        self.sep = " "  # Default separator
+        self.last_response_content = ""
+        
+        # Models and processing
+        self.asr = models.asr
+        self.tokenizer = models.tokenizer
+        self.diarization = models.diarization
+        self.ffmpeg_process = self.start_ffmpeg_decoder()
+        self.transcription_queue = asyncio.Queue() if self.args.transcription else None
+        self.diarization_queue = asyncio.Queue() if self.args.diarization else None
+        self.pcm_buffer = bytearray()
+        
+        # Initialize transcription engine if enabled
+        if self.args.transcription:
+            self.online = online_factory(self.args, models.asr, models.tokenizer)
+
+    def convert_pcm_to_float(self, pcm_buffer):
+        """Convert PCM buffer in s16le format to normalized NumPy array."""
+        return np.frombuffer(pcm_buffer, dtype=np.int16).astype(np.float32) / 32768.0
+
+    def start_ffmpeg_decoder(self):
+        """Start FFmpeg process for WebM to PCM conversion."""
+        return (ffmpeg.input("pipe:0", format="webm")
+                .output("pipe:1", format="s16le", acodec="pcm_s16le", 
+                        ac=self.channels, ar=str(self.sample_rate))
+                .run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True))
+
+    async def restart_ffmpeg(self):
+        """Restart the FFmpeg process after failure."""
+        if self.ffmpeg_process:
+            try:
+                self.ffmpeg_process.kill()
+                await asyncio.get_event_loop().run_in_executor(None, self.ffmpeg_process.wait)
+            except Exception as e:
+                logger.warning(f"Error killing FFmpeg process: {e}")
+            self.ffmpeg_process = self.start_ffmpeg_decoder()
+            self.pcm_buffer = bytearray()
+
+    async def update_transcription(self, new_tokens, buffer, end_buffer, full_transcription, sep):
+        """Thread-safe update of transcription with new data."""
+        async with self.lock:
+            self.tokens.extend(new_tokens)
+            self.buffer_transcription = buffer
+            self.end_buffer = end_buffer
+            self.full_transcription = full_transcription
+            self.sep = sep
+            
+    async def update_diarization(self, end_attributed_speaker, buffer_diarization=""):
+        """Thread-safe update of diarization with new data."""
+        async with self.lock:
+            self.end_attributed_speaker = end_attributed_speaker
+            if buffer_diarization:
+                self.buffer_diarization = buffer_diarization
+            
+    async def add_dummy_token(self):
+        """Placeholder token when no transcription is available."""
+        async with self.lock:
+            current_time = time() - self.beg_loop
+            self.tokens.append(ASRToken(
+                start=current_time, end=current_time + 1,
+                text=".", speaker=-1, is_dummy=True
+            ))
+            
+    async def get_current_state(self):
+        """Get current state."""
+        async with self.lock:
+            current_time = time()
+            
+            # Calculate remaining times
+            remaining_transcription = 0
+            if self.end_buffer > 0:
+                remaining_transcription = max(0, round(current_time - self.beg_loop - self.end_buffer, 2))
+                
+            remaining_diarization = 0
+            if self.tokens:
+                latest_end = max(self.end_buffer, self.tokens[-1].end if self.tokens else 0)
+                remaining_diarization = max(0, round(latest_end - self.end_attributed_speaker, 2))
+                
+            return {
+                "tokens": self.tokens.copy(),
+                "buffer_transcription": self.buffer_transcription,
+                "buffer_diarization": self.buffer_diarization,
+                "end_buffer": self.end_buffer,
+                "end_attributed_speaker": self.end_attributed_speaker,
+                "sep": self.sep,
+                "remaining_time_transcription": remaining_transcription,
+                "remaining_time_diarization": remaining_diarization
+            }
+            
+    async def reset(self):
+        """Reset all state variables to initial values."""
+        async with self.lock:
+            self.tokens = []
+            self.buffer_transcription = self.buffer_diarization = ""
+            self.end_buffer = self.end_attributed_speaker = 0
+            self.full_transcription = self.last_response_content = ""
+            self.beg_loop = time()
+
+    async def ffmpeg_stdout_reader(self):
+        """Read audio data from FFmpeg stdout and process it."""
+        loop = asyncio.get_event_loop()
+        beg = time()
+        
+        while True:
+            try:
+                # Calculate buffer size based on elapsed time
+                elapsed_time = math.floor((time() - beg) * 10) / 10  # Round to 0.1 sec
+                buffer_size = max(int(32000 * elapsed_time), 4096)
+                beg = time()
+
+                # Read chunk with timeout
+                try:
+                    chunk = await asyncio.wait_for(
+                        loop.run_in_executor(None, self.ffmpeg_process.stdout.read, buffer_size),
+                        timeout=15.0
+                    )
+                except asyncio.TimeoutError:
+                    logger.warning("FFmpeg read timeout. Restarting...")
+                    await self.restart_ffmpeg()
+                    beg = time()
+                    continue
+
+                if not chunk:
+                    logger.info("FFmpeg stdout closed.")
+                    break
+                    
+                self.pcm_buffer.extend(chunk)
+                        
+                # Send to diarization if enabled
+                if self.args.diarization and self.diarization_queue:
+                    await self.diarization_queue.put(
+                        self.convert_pcm_to_float(self.pcm_buffer).copy()
+                    )
+
+                # Process when we have enough data
+                if len(self.pcm_buffer) >= self.bytes_per_sec:
+                    if len(self.pcm_buffer) > self.max_bytes_per_sec:
+                        logger.warning(
+                            f"Audio buffer too large: {len(self.pcm_buffer) / self.bytes_per_sec:.2f}s. "
+                            f"Consider using a smaller model."
+                        )
+
+                    # Process audio chunk
+                    pcm_array = self.convert_pcm_to_float(self.pcm_buffer[:self.max_bytes_per_sec])
+                    self.pcm_buffer = self.pcm_buffer[self.max_bytes_per_sec:]
+                    
+                    # Send to transcription if enabled
+                    if self.args.transcription and self.transcription_queue:
+                        await self.transcription_queue.put(pcm_array.copy())
+                    
+                    # Sleep if no processing is happening
+                    if not self.args.transcription and not self.args.diarization:
+                        await asyncio.sleep(0.1)
+                    
+            except Exception as e:
+                logger.warning(f"Exception in ffmpeg_stdout_reader: {e}")
+                logger.warning(f"Traceback: {traceback.format_exc()}")
+                break
+
+    async def transcription_processor(self):
+        """Process audio chunks for transcription."""
+        self.full_transcription = ""
+        self.sep = self.online.asr.sep
+        
+        while True:
+            try:
+                pcm_array = await self.transcription_queue.get()
+                
+                logger.info(f"{len(self.online.audio_buffer) / self.online.SAMPLING_RATE} seconds of audio to process.")
+                
+                # Process transcription
+                self.online.insert_audio_chunk(pcm_array)
+                new_tokens = self.online.process_iter()
+                
+                if new_tokens:
+                    self.full_transcription += self.sep.join([t.text for t in new_tokens])
+                    
+                # Get buffer information
+                _buffer = self.online.get_buffer()
+                buffer = _buffer.text
+                end_buffer = _buffer.end if _buffer.end else (
+                    new_tokens[-1].end if new_tokens else 0
+                )
+                
+                # Avoid duplicating content
+                if buffer in self.full_transcription:
+                    buffer = ""
+                    
+                await self.update_transcription(
+                    new_tokens, buffer, end_buffer, self.full_transcription, self.sep
+                )
+                
+            except Exception as e:
+                logger.warning(f"Exception in transcription_processor: {e}")
+                logger.warning(f"Traceback: {traceback.format_exc()}")
+            finally:
+                self.transcription_queue.task_done()
+
+    async def diarization_processor(self, diarization_obj):
+        """Process audio chunks for speaker diarization."""
+        buffer_diarization = ""
+        
+        while True:
+            try:
+                pcm_array = await self.diarization_queue.get()
+                
+                # Process diarization
+                await diarization_obj.diarize(pcm_array)
+                
+                # Get current state and update speakers
+                state = await self.get_current_state()
+                new_end = diarization_obj.assign_speakers_to_tokens(
+                    state["end_attributed_speaker"], state["tokens"]
+                )
+                
+                await self.update_diarization(new_end, buffer_diarization)
+                
+            except Exception as e:
+                logger.warning(f"Exception in diarization_processor: {e}")
+                logger.warning(f"Traceback: {traceback.format_exc()}")
+            finally:
+                self.diarization_queue.task_done()
+
+    async def results_formatter(self):
+        """Format processing results for output."""
+        while True:
+            try:
+                # Get current state
+                state = await self.get_current_state()
+                tokens = state["tokens"]
+                buffer_transcription = state["buffer_transcription"]
+                buffer_diarization = state["buffer_diarization"]
+                end_attributed_speaker = state["end_attributed_speaker"]
+                sep = state["sep"]
+                
+                # Add dummy tokens if needed
+                if (not tokens or tokens[-1].is_dummy) and not self.args.transcription and self.args.diarization:
+                    await self.add_dummy_token()
+                    sleep(0.5)
+                    state = await self.get_current_state()
+                    tokens = state["tokens"]
+                
+                # Format output
+                previous_speaker = -1
+                lines = []
+                last_end_diarized = 0
+                undiarized_text = []
+                
+                # Process each token
+                for token in tokens:
+                    speaker = token.speaker
+                    
+                    # Handle diarization
+                    if self.args.diarization:
+                        if (speaker in [-1, 0]) and token.end >= end_attributed_speaker:
+                            undiarized_text.append(token.text)
+                            continue
+                        elif (speaker in [-1, 0]) and token.end < end_attributed_speaker:
+                            speaker = previous_speaker
+                        if speaker not in [-1, 0]:
+                            last_end_diarized = max(token.end, last_end_diarized)
+
+                    # Group by speaker
+                    if speaker != previous_speaker or not lines:
+                        lines.append({
+                            "speaker": speaker,
+                            "text": token.text,
+                            "beg": format_time(token.start),
+                            "end": format_time(token.end),
+                            "diff": round(token.end - last_end_diarized, 2)
+                        })
+                        previous_speaker = speaker
+                    elif token.text:  # Only append if text isn't empty
+                        lines[-1]["text"] += sep + token.text
+                        lines[-1]["end"] = format_time(token.end)
+                        lines[-1]["diff"] = round(token.end - last_end_diarized, 2)
+                
+                # Handle undiarized text
+                if undiarized_text:
+                    combined = sep.join(undiarized_text)
+                    if buffer_transcription:
+                        combined += sep
+                    await self.update_diarization(end_attributed_speaker, combined)
+                    buffer_diarization = combined
+                
+                # Create response object
+                if not lines:
+                    lines = [{
+                        "speaker": 1,
+                        "text": "",
+                        "beg": format_time(0),
+                        "end": format_time(tokens[-1].end if tokens else 0),
+                        "diff": 0
+                    }]
+                
+                response = {
+                    "lines": lines, 
+                    "buffer_transcription": buffer_transcription,
+                    "buffer_diarization": buffer_diarization,
+                    "remaining_time_transcription": state["remaining_time_transcription"],
+                    "remaining_time_diarization": state["remaining_time_diarization"]
+                }
+                
+                # Only yield if content has changed
+                response_content = ' '.join([f"{line['speaker']} {line['text']}" for line in lines]) + \
+                                  f" | {buffer_transcription} | {buffer_diarization}"
+                
+                if response_content != self.last_response_content and (lines or buffer_transcription or buffer_diarization):
+                    yield response
+                    self.last_response_content = response_content
+                
+                await asyncio.sleep(0.1)  # Avoid overwhelming the client
+                
+            except Exception as e:
+                logger.warning(f"Exception in results_formatter: {e}")
+                logger.warning(f"Traceback: {traceback.format_exc()}")
+                await asyncio.sleep(0.5)  # Back off on error
+                
+    async def create_tasks(self):
+        """Create and start processing tasks."""
+            
+        tasks = []    
+        if self.args.transcription and self.online:
+            tasks.append(asyncio.create_task(self.transcription_processor()))
+            
+        if self.args.diarization and self.diarization:
+            tasks.append(asyncio.create_task(self.diarization_processor(self.diarization)))
+        
+        tasks.append(asyncio.create_task(self.ffmpeg_stdout_reader()))
+        self.tasks = tasks
+        
+        return self.results_formatter()
+        
+    async def cleanup(self):
+        """Clean up resources when processing is complete."""
+        for task in self.tasks:
+            task.cancel()
+            
+        try:
+            await asyncio.gather(*self.tasks, return_exceptions=True)
+            self.ffmpeg_process.stdin.close()
+            self.ffmpeg_process.wait()
+        except Exception as e:
+            logger.warning(f"Error during cleanup: {e}")
+            
+        if self.args.diarization and hasattr(self, 'diarization'):
+            self.diarization.close()
+
+    async def process_audio(self, message):
+        """Process incoming audio data."""
+        try:
+            self.ffmpeg_process.stdin.write(message)
+            self.ffmpeg_process.stdin.flush()
+        except (BrokenPipeError, AttributeError) as e:
+            logger.warning(f"Error writing to FFmpeg: {e}. Restarting...")
+            await self.restart_ffmpeg()
+            self.ffmpeg_process.stdin.write(message)
+            self.ffmpeg_process.stdin.flush()

core.py

@@ -0,0 +1,61 @@
+from whisper_streaming_custom.whisper_online import backend_factory, warmup_asr
+from argparse import Namespace, ArgumentParser
+
+class WhisperLiveKit:
+    _instance = None
+    _initialized = False
+    
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+    
+    def __init__(self, args=None, **kwargs):
+        if WhisperLiveKit._initialized:
+            return
+            
+        if args is None:
+            args = Namespace(
+                host="localhost",
+                port=8000,
+                warmup_file=None,
+                confidence_validation=False,
+                diarization=False,
+                transcription=True,
+                min_chunk_size=0.5,
+                model="base",
+                model_cache_dir=None,
+                model_dir=None,
+                lan="auto",
+                task="transcribe",
+                backend="faster-whisper",
+                vac=False,
+                vac_chunk_size=0.04,
+                vad=True,
+                buffer_trimming="sentence",
+                buffer_trimming_sec=1.0,
+                log_level="INFO"
+            )
+        
+        self.args = args
+        
+        self.asr = None
+        self.tokenizer = None
+        self.diarization = None
+        
+        if self.args.transcription:
+            self.asr, self.tokenizer = backend_factory(self.args)
+            warmup_asr(self.asr, self.args.warmup_file)
+
+        if self.args.diarization:
+            from diarization.diarization_online import DiartDiarization
+            self.diarization = DiartDiarization()
+            
+        WhisperLiveKit._initialized = True
+
+    def web_interface(self):
+        import pkg_resources
+        html_path = pkg_resources.resource_filename('whisperlivekit', 'web/live_transcription.html')
+        with open(html_path, "r", encoding="utf-8") as f:
+            html = f.read()
+        return html

diarization/diarization_online.py

@@ -0,0 +1,153 @@
+import asyncio
+import re
+import threading
+import numpy as np
+import logging
+
+
+from diart import SpeakerDiarization, SpeakerDiarizationConfig
+from diart.inference import StreamingInference
+from diart.sources import AudioSource
+from timed_objects import SpeakerSegment
+from diart.sources import MicrophoneAudioSource
+from rx.core import Observer
+from typing import Tuple, Any, List
+from pyannote.core import Annotation
+
+logger = logging.getLogger(__name__)
+
+def extract_number(s: str) -> int:
+    m = re.search(r'\d+', s)
+    return int(m.group()) if m else None
+
+class DiarizationObserver(Observer):
+    """Observer that logs all data emitted by the diarization pipeline and stores speaker segments."""
+    
+    def __init__(self):
+        self.speaker_segments = []
+        self.processed_time = 0
+        self.segment_lock = threading.Lock()
+    
+    def on_next(self, value: Tuple[Annotation, Any]):
+        annotation, audio = value
+        
+        logger.debug("\n--- New Diarization Result ---")
+        
+        duration = audio.extent.end - audio.extent.start
+        logger.debug(f"Audio segment: {audio.extent.start:.2f}s - {audio.extent.end:.2f}s (duration: {duration:.2f}s)")
+        logger.debug(f"Audio shape: {audio.data.shape}")
+        
+        with self.segment_lock:
+            if audio.extent.end > self.processed_time:
+                self.processed_time = audio.extent.end            
+            if annotation and len(annotation._labels) > 0:
+                logger.debug("\nSpeaker segments:")
+                for speaker, label in annotation._labels.items():
+                    for start, end in zip(label.segments_boundaries_[:-1], label.segments_boundaries_[1:]):
+                        print(f"  {speaker}: {start:.2f}s-{end:.2f}s")
+                        self.speaker_segments.append(SpeakerSegment(
+                            speaker=speaker,
+                            start=start,
+                            end=end
+                        ))
+            else:
+                logger.debug("\nNo speakers detected in this segment")
+                
+    def get_segments(self) -> List[SpeakerSegment]:
+        """Get a copy of the current speaker segments."""
+        with self.segment_lock:
+            return self.speaker_segments.copy()
+    
+    def clear_old_segments(self, older_than: float = 30.0):
+        """Clear segments older than the specified time."""
+        with self.segment_lock:
+            current_time = self.processed_time
+            self.speaker_segments = [
+                segment for segment in self.speaker_segments 
+                if current_time - segment.end < older_than
+            ]
+    
+    def on_error(self, error):
+        """Handle an error in the stream."""
+        logger.debug(f"Error in diarization stream: {error}")
+        
+    def on_completed(self):
+        """Handle the completion of the stream."""
+        logger.debug("Diarization stream completed")
+
+
+class WebSocketAudioSource(AudioSource):
+    """
+    Custom AudioSource that blocks in read() until close() is called.
+    Use push_audio() to inject PCM chunks.
+    """
+    def __init__(self, uri: str = "websocket", sample_rate: int = 16000):
+        super().__init__(uri, sample_rate)
+        self._closed = False
+        self._close_event = threading.Event()
+
+    def read(self):
+        self._close_event.wait()
+
+    def close(self):
+        if not self._closed:
+            self._closed = True
+            self.stream.on_completed()
+            self._close_event.set()
+
+    def push_audio(self, chunk: np.ndarray):
+        if not self._closed:
+            new_audio = np.expand_dims(chunk, axis=0)
+            logger.debug('Add new chunk with shape:', new_audio.shape)
+            self.stream.on_next(new_audio)
+
+
+class DiartDiarization:
+    def __init__(self, sample_rate: int = 16000, config : SpeakerDiarizationConfig = None, use_microphone: bool = False):
+        self.pipeline = SpeakerDiarization(config=config)        
+        self.observer = DiarizationObserver()
+        
+        if use_microphone:
+            self.source = MicrophoneAudioSource()
+            self.custom_source = None
+        else:
+            self.custom_source = WebSocketAudioSource(uri="websocket_source", sample_rate=sample_rate)
+            self.source = self.custom_source
+            
+        self.inference = StreamingInference(
+            pipeline=self.pipeline,
+            source=self.source,
+            do_plot=False,
+            show_progress=False,
+        )
+        self.inference.attach_observers(self.observer)
+        asyncio.get_event_loop().run_in_executor(None, self.inference)
+
+    async def diarize(self, pcm_array: np.ndarray):
+        """
+        Process audio data for diarization.
+        Only used when working with WebSocketAudioSource.
+        """
+        if self.custom_source:
+            self.custom_source.push_audio(pcm_array)            
+        self.observer.clear_old_segments()        
+        return self.observer.get_segments()
+
+    def close(self):
+        """Close the audio source."""
+        if self.custom_source:
+            self.custom_source.close()
+
+    def assign_speakers_to_tokens(self, end_attributed_speaker, tokens: list) -> float:
+        """
+        Assign speakers to tokens based on timing overlap with speaker segments.
+        Uses the segments collected by the observer.
+        """
+        segments = self.observer.get_segments()
+        
+        for token in tokens:
+            for segment in segments:
+                if not (segment.end <= token.start or segment.start >= token.end):
+                    token.speaker = extract_number(segment.speaker) + 1
+                    end_attributed_speaker = max(token.end, end_attributed_speaker)
+        return end_attributed_speaker

docker-compose.yml

@@ -0,0 +1,36 @@
+version: '3.8'
+
+services:
+  websocket-server:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - "8000:8000"
+    environment:
+      - PYTHONUNBUFFERED=1
+    volumes:
+      - .:/app
+    networks:
+      - app-network
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  frontend:
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile
+    ports:
+      - "80:80"
+    depends_on:
+      websocket-server:
+        condition: service_healthy
+    networks:
+      - app-network
+
+networks:
+  app-network:
+    driver: bridge

main.py

@@ -0,0 +1,275 @@
+from contextlib import asynccontextmanager
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+import asyncio
+import logging
+import os
+import traceback
+import argparse
+import uvicorn
+
+from core import WhisperLiveKit
+from audio_processor import AudioProcessor
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logging.getLogger().setLevel(logging.WARNING)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+kit = None
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global kit
+    kit = WhisperLiveKit()
+    yield
+
+app = FastAPI(lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Allows all origins
+    allow_credentials=True,
+    allow_methods=["*"],  # Allows all methods
+    allow_headers=["*"],  # Allows all headers
+)
+
+# Mount static files
+app.mount("/static", StaticFiles(directory="static"), name="static")
+
+
+
+@app.get("/")
+async def read_root():
+    return FileResponse("static/index.html")
+
+@app.get("/health")
+async def health_check():
+    return JSONResponse({"status": "healthy"})
+
+async def handle_websocket_results(websocket, results_generator):
+    """Consumes results from the audio processor and sends them via WebSocket."""
+    try:
+        async for response in results_generator:
+            try:
+                logger.debug(f"Sending response: {response}")
+                if isinstance(response, dict):
+                    # Ensure the response has a consistent format
+                    if 'buffer_transcription' in response:
+                        await websocket.send_json({
+                            'buffer_transcription': response['buffer_transcription']
+                        })
+                    elif 'full_transcription' in response:
+                        await websocket.send_json({
+                            'full_transcription': response['full_transcription']
+                        })
+                    else:
+                        await websocket.send_json(response)
+                else:
+                    # If response is not a dict, wrap it in a text field
+                    await websocket.send_json({"text": str(response)})
+            except Exception as e:
+                logger.error(f"Error sending message: {e}")
+                logger.error(f"Traceback: {traceback.format_exc()}")
+                raise
+    except Exception as e:
+        logger.warning(f"Error in WebSocket results handler: {e}")
+        logger.warning(f"Traceback: {traceback.format_exc()}")
+
+@app.websocket("/asr")
+async def websocket_endpoint(websocket: WebSocket):
+    logger.info("New WebSocket connection request")
+    audio_processor = None
+    websocket_task = None
+
+    try:
+        await websocket.accept()
+        logger.info("WebSocket connection accepted")
+        
+        audio_processor = AudioProcessor()
+        results_generator = await audio_processor.create_tasks()
+        websocket_task = asyncio.create_task(handle_websocket_results(websocket, results_generator))
+
+        while True:
+            try:
+                message = await websocket.receive_bytes()
+                logger.debug(f"Received audio chunk of size: {len(message)}")
+                await audio_processor.process_audio(message)
+            except WebSocketDisconnect:
+                logger.warning("WebSocket disconnected.")
+                break
+            except Exception as e:
+                logger.error(f"Error processing audio chunk: {e}")
+                logger.error(f"Traceback: {traceback.format_exc()}")
+                break
+
+    except WebSocketDisconnect:
+        logger.warning("WebSocket disconnected during setup.")
+    except Exception as e:
+        logger.error(f"Error in WebSocket endpoint: {e}")
+        logger.error(f"Traceback: {traceback.format_exc()}")
+    finally:
+        if websocket_task:
+            websocket_task.cancel()
+            try:
+                await websocket_task
+            except asyncio.CancelledError:
+                pass
+        if audio_processor:
+            await audio_processor.cleanup()
+        logger.info("WebSocket endpoint cleaned up.")
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Whisper FastAPI Online Server")
+    parser.add_argument(
+        "--host",
+        type=str,
+        default="localhost",
+        help="The host address to bind the server to.",
+    )
+    parser.add_argument(
+        "--port", type=int, default=8000, help="The port number to bind the server to."
+    )
+    parser.add_argument(
+        "--warmup-file",
+        type=str,
+        default=None,
+        dest="warmup_file",
+        help="""
+        The path to a speech audio wav file to warm up Whisper so that the very first chunk processing is fast.
+        If not set, uses https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav.
+        If False, no warmup is performed.
+        """,
+    )
+
+    parser.add_argument(
+        "--confidence-validation",
+        action="store_true",
+        help="Accelerates validation of tokens using confidence scores. Transcription will be faster but punctuation might be less accurate.",
+    )
+
+    parser.add_argument(
+        "--diarization",
+        action="store_true",
+        default=False,
+        help="Enable speaker diarization.",
+    )
+
+    parser.add_argument(
+        "--no-transcription",
+        action="store_true",
+        help="Disable transcription to only see live diarization results.",
+    )
+    
+    parser.add_argument(
+        "--min-chunk-size",
+        type=float,
+        default=0.5,
+        help="Minimum audio chunk size in seconds. It waits up to this time to do processing. If the processing takes shorter time, it waits, otherwise it processes the whole segment that was received by this time.",
+    )
+    
+    parser.add_argument(
+        "--model",
+        type=str,
+        default="tiny",
+        help="Name size of the Whisper model to use (default: tiny). Suggested values: tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large,large-v3-turbo. The model is automatically downloaded from the model hub if not present in model cache dir.",
+    )
+    
+    parser.add_argument(
+        "--model_cache_dir",
+        type=str,
+        default=None,
+        help="Overriding the default model cache dir where models downloaded from the hub are saved",
+    )
+    parser.add_argument(
+        "--model_dir",
+        type=str,
+        default=None,
+        help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.",
+    )
+    parser.add_argument(
+        "--lan",
+        "--language",
+        type=str,
+        default="en",
+        help="Source language code, e.g. en,de,cs, or 'auto' for language detection.",
+    )
+    parser.add_argument(
+        "--task",
+        type=str,
+        default="transcribe",
+        choices=["transcribe", "translate"],
+        help="Transcribe or translate.",
+    )
+    parser.add_argument(
+        "--backend",
+        type=str,
+        default="faster-whisper",
+        choices=["faster-whisper", "whisper_timestamped", "mlx-whisper", "openai-api"],
+        help="Load only this backend for Whisper processing.",
+    )
+    parser.add_argument(
+        "--vac",
+        action="store_true",
+        default=False,
+        help="Use VAC = voice activity controller. Recommended. Requires torch.",
+    )
+    parser.add_argument(
+        "--vac-chunk-size", type=float, default=0.04, help="VAC sample size in seconds."
+    )
+    parser.add_argument(
+        "--no-vad",
+        action="store_true",
+        default=False,
+        help="Disable VAD = voice activity detection. Not recommended.",
+    )
+    parser.add_argument(
+        "--buffer_trimming",
+        type=str,
+        default="sentence",
+        choices=["sentence", "segment"],
+        help="Buffer trimming strategy.",
+    )
+    parser.add_argument(
+        "--buffer_trimming_sec",
+        type=float,
+        default=1.0,
+        help="Buffer trimming length in seconds.",
+    )
+    parser.add_argument(
+        "-l",
+        "--log-level",
+        type=str,
+        default="INFO",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+        help="Set the logging level.",
+    )
+
+    args = parser.parse_args()
+    
+    args.transcription = not args.no_transcription
+    args.vad = not args.no_vad    
+    delattr(args, 'no_transcription')
+    delattr(args, 'no_vad')
+    
+    return args
+
+def main():
+    args = parse_args()
+    
+    # Initialize WhisperLiveKit with parsed arguments
+    kit = WhisperLiveKit(args=args)
+    
+    # Start the server
+    uvicorn.run(
+        "main:app",
+        host=args.host,
+        port=args.port,
+        log_level=args.log_level.lower()
+    )
+
+if __name__ == "__main__":
+    main() 

requirements.txt

@@ -0,0 +1,15 @@
+fastapi>=0.68.0
+uvicorn[standart]
+python-multipart>=0.0.5
+numpy>=1.21.0
+ffmpeg-python>=0.2.0
+torch>=2.0.0
+torchaudio>=2.0.0
+faster-whisper>=0.9.0
+websockets>=10.0
+pydantic>=1.8.0
+python-dotenv>=0.19.0
+setuptools>=65.5.1
+librosa>=0.10.0
+mosestokenizer
+hf_xet

setup.py

@@ -0,0 +1,22 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="whisperlivekit",
+    version="0.1.0",
+    packages=find_packages(),
+    install_requires=[
+        "fastapi>=0.68.0",
+        "uvicorn>=0.15.0",
+        "python-multipart>=0.0.5",
+        "numpy>=1.21.0",
+        "ffmpeg-python>=0.2.0",
+        "torch>=2.0.0",
+        "torchaudio>=2.0.0",
+        "faster-whisper>=0.9.0",
+        "websockets>=10.0",
+        "pydantic>=1.8.0",
+        "python-dotenv>=0.19.0",
+        "setuptools>=65.5.1",
+    ],
+    python_requires=">=3.10",
+) 

static/index.html

@@ -0,0 +1,248 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Whisper Live Transcription</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            max-width: 800px;
+            margin: 0 auto;
+            padding: 20px;
+            background-color: #f5f5f5;
+        }
+        .container {
+            background-color: white;
+            padding: 20px;
+            border-radius: 8px;
+            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+        }
+        .controls {
+            margin: 20px 0;
+        }
+        button {
+            padding: 10px 20px;
+            margin: 5px;
+            border: none;
+            border-radius: 4px;
+            background-color: #007bff;
+            color: white;
+            cursor: pointer;
+        }
+        button:disabled {
+            background-color: #cccccc;
+            cursor: not-allowed;
+        }
+        #clearBtn {
+            background-color: #dc3545;
+        }
+        #clearBtn:hover {
+            background-color: #c82333;
+        }
+        #transcription {
+            margin-top: 20px;
+            padding: 15px;
+            border: 1px solid #ddd;
+            border-radius: 4px;
+            min-height: 100px;
+            white-space: pre-wrap;
+            font-family: monospace;
+            line-height: 1.5;
+        }
+        .status {
+            margin: 10px 0;
+            padding: 10px;
+            border-radius: 4px;
+        }
+        .config {
+            margin: 10px 0;
+            padding: 10px;
+            background-color: #f8f9fa;
+            border-radius: 4px;
+        }
+        .config input {
+            margin-left: 10px;
+            padding: 5px;
+            border: 1px solid #ddd;
+            border-radius: 4px;
+        }
+        .connected {
+            background-color: #d4edda;
+            color: #155724;
+        }
+        .disconnected {
+            background-color: #f8d7da;
+            color: #721c24;
+        }
+        #error-message {
+            color: #721c24;
+            margin: 10px 0;
+            padding: 10px;
+            background-color: #f8d7da;
+            border-radius: 4px;
+            display: none;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>Whisper Live Transcription</h1>
+        <div class="config">
+            <label for="wsUrl">WebSocket URL:</label>
+            <input type="text" id="wsUrl" value="ws://localhost:7860/asr" style="width: 300px;">
+        </div>
+        <div id="status" class="status disconnected">Disconnected</div>
+        <div id="error-message"></div>
+        <div class="controls">
+            <button id="startBtn">Start Recording</button>
+            <button id="stopBtn" disabled>Stop Recording</button>
+            <button id="reconnectBtn">Reconnect</button>
+            <button id="clearBtn">Clear Transcription</button>
+        </div>
+        <div id="transcription"></div>
+    </div>
+
+    <script>
+        let ws = null;
+        let mediaRecorder = null;
+        let audioChunks = [];
+        let reconnectAttempts = 0;
+        const maxReconnectAttempts = 5;
+        const reconnectDelay = 2000; // 2 seconds
+
+        const startBtn = document.getElementById('startBtn');
+        const stopBtn = document.getElementById('stopBtn');
+        const reconnectBtn = document.getElementById('reconnectBtn');
+        const clearBtn = document.getElementById('clearBtn');
+        const wsUrlInput = document.getElementById('wsUrl');
+        const statusDiv = document.getElementById('status');
+        const transcriptionDiv = document.getElementById('transcription');
+        const errorMessageDiv = document.getElementById('error-message');
+
+        function showError(message) {
+            errorMessageDiv.textContent = message;
+            errorMessageDiv.style.display = 'block';
+        }
+
+        function hideError() {
+            errorMessageDiv.style.display = 'none';
+        }
+
+        function updateStatus(connected) {
+            statusDiv.textContent = connected ? 'Connected' : 'Disconnected';
+            statusDiv.className = `status ${connected ? 'connected' : 'disconnected'}`;
+            reconnectBtn.disabled = connected;
+        }
+
+        function connectWebSocket() {
+            if (ws) {
+                ws.close();
+            }
+
+            const wsUrl = wsUrlInput.value;
+            console.log('Attempting to connect to:', wsUrl);
+            ws = new WebSocket(wsUrl);
+            
+            ws.onopen = () => {
+                console.log('WebSocket connection established');
+                updateStatus(true);
+                startBtn.disabled = false;
+                stopBtn.disabled = true;
+                hideError();
+                reconnectAttempts = 0;
+            };
+
+            ws.onclose = (event) => {
+                console.log('WebSocket connection closed:', event.code, event.reason);
+                updateStatus(false);
+                startBtn.disabled = true;
+                stopBtn.disabled = true;
+                
+                if (event.code === 1006) {
+                    showError('Connection lost. Click "Reconnect" to try again.');
+                }
+            };
+
+            ws.onerror = (error) => {
+                console.error('WebSocket error:', error);
+                updateStatus(false);
+                showError('Connection error occurred. Click "Reconnect" to try again.');
+            };
+
+            ws.onmessage = (event) => {
+                try {
+                    console.log('Received message:', event.data);
+                    const response = JSON.parse(event.data);
+                    console.log('Parsed response:', response);
+                    
+                    if (response.text) {
+                        console.log('Adding text:', response.text);
+                        transcriptionDiv.textContent += response.text + '\n';
+                    } else if (response.partial) {
+                        console.log('Adding partial text:', response.partial);
+                        transcriptionDiv.textContent = transcriptionDiv.textContent.replace(/[^.!?]+$/, '') + response.partial;
+                    } else if (response.error) {
+                        console.error('Server error:', response.error);
+                        showError('Server error: ' + response.error);
+                    } else if (response.buffer_transcription) {
+                        console.log('Adding buffer transcription:', response.buffer_transcription);
+                        transcriptionDiv.textContent += response.buffer_transcription + '\n';
+                    } else if (response.full_transcription) {
+                        console.log('Adding full transcription:', response.full_transcription);
+                        transcriptionDiv.textContent += response.full_transcription + '\n';
+                    } else if (typeof response === 'string') {
+                        console.log('Adding raw text:', response);
+                        transcriptionDiv.textContent += response + '\n';
+                    }
+                } catch (error) {
+                    console.error('Error parsing message:', error);
+                    console.error('Raw message:', event.data);
+                }
+            };
+        }
+
+        async function startRecording() {
+            try {
+                const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+                mediaRecorder = new MediaRecorder(stream);
+                audioChunks = [];
+
+                mediaRecorder.ondataavailable = (event) => {
+                    if (event.data.size > 0 && ws && ws.readyState === WebSocket.OPEN) {
+                        ws.send(event.data);
+                    }
+                };
+
+                mediaRecorder.start(100); // Send chunks every 100ms
+                startBtn.disabled = true;
+                stopBtn.disabled = false;
+            } catch (error) {
+                console.error('Error accessing microphone:', error);
+                showError('Error accessing microphone. Please ensure you have granted microphone permissions.');
+            }
+        }
+
+        function stopRecording() {
+            if (mediaRecorder && mediaRecorder.state !== 'inactive') {
+                mediaRecorder.stop();
+                mediaRecorder.stream.getTracks().forEach(track => track.stop());
+                startBtn.disabled = false;
+                stopBtn.disabled = true;
+            }
+        }
+
+        function clearTranscription() {
+            transcriptionDiv.textContent = '';
+        }
+
+        startBtn.addEventListener('click', startRecording);
+        stopBtn.addEventListener('click', stopRecording);
+        reconnectBtn.addEventListener('click', connectWebSocket);
+        clearBtn.addEventListener('click', clearTranscription);
+
+        // Connect to WebSocket when page loads
+        connectWebSocket();
+    </script>
+</body>
+</html> 

timed_objects.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+from typing import Optional
+
+@dataclass
+class TimedText:
+    start: Optional[float]
+    end: Optional[float]
+    text: Optional[str] = ''
+    speaker: Optional[int] = -1
+    probability: Optional[float] = None
+    is_dummy: Optional[bool] = False
+
+@dataclass
+class ASRToken(TimedText):
+    def with_offset(self, offset: float) -> "ASRToken":
+        """Return a new token with the time offset added."""
+        return ASRToken(self.start + offset, self.end + offset, self.text, self.speaker, self.probability)
+
+@dataclass
+class Sentence(TimedText):
+    pass
+
+@dataclass
+class Transcript(TimedText):
+    pass
+
+@dataclass
+class SpeakerSegment(TimedText):
+    pass

whisper_streaming_custom/backends.py

@@ -0,0 +1,295 @@
+import sys
+import logging
+import io
+import soundfile as sf
+import math
+try: 
+    import torch
+except ImportError: 
+    torch = None
+from typing import List
+import numpy as np
+from timed_objects import ASRToken
+
+logger = logging.getLogger(__name__)
+
+class ASRBase:
+    sep = " "  # join transcribe words with this character (" " for whisper_timestamped,
+              # "" for faster-whisper because it emits the spaces when needed)
+
+    def __init__(self, lan, modelsize=None, cache_dir=None, model_dir=None, logfile=sys.stderr):
+        self.logfile = logfile
+        self.transcribe_kargs = {}
+        if lan == "auto":
+            self.original_language = None
+        else:
+            self.original_language = lan
+        self.model = self.load_model(modelsize, cache_dir, model_dir)
+
+    def with_offset(self, offset: float) -> ASRToken:
+        # This method is kept for compatibility (typically you will use ASRToken.with_offset)
+        return ASRToken(self.start + offset, self.end + offset, self.text)
+
+    def __repr__(self):
+        return f"ASRToken(start={self.start:.2f}, end={self.end:.2f}, text={self.text!r})"
+
+    def load_model(self, modelsize, cache_dir, model_dir):
+        raise NotImplementedError("must be implemented in the child class")
+
+    def transcribe(self, audio, init_prompt=""):
+        raise NotImplementedError("must be implemented in the child class")
+
+    def use_vad(self):
+        raise NotImplementedError("must be implemented in the child class")
+
+
+class WhisperTimestampedASR(ASRBase):
+    """Uses whisper_timestamped as the backend."""
+    sep = " "
+
+    def load_model(self, modelsize=None, cache_dir=None, model_dir=None):
+        import whisper
+        import whisper_timestamped
+        from whisper_timestamped import transcribe_timestamped
+
+        self.transcribe_timestamped = transcribe_timestamped
+        if model_dir is not None:
+            logger.debug("ignoring model_dir, not implemented")
+        return whisper.load_model(modelsize, download_root=cache_dir)
+
+    def transcribe(self, audio, init_prompt=""):
+        result = self.transcribe_timestamped(
+            self.model,
+            audio,
+            language=self.original_language,
+            initial_prompt=init_prompt,
+            verbose=None,
+            condition_on_previous_text=True,
+            **self.transcribe_kargs,
+        )
+        return result
+
+    def ts_words(self, r) -> List[ASRToken]:
+        """
+        Converts the whisper_timestamped result to a list of ASRToken objects.
+        """
+        tokens = []
+        for segment in r["segments"]:
+            for word in segment["words"]:
+                token = ASRToken(word["start"], word["end"], word["text"])
+                tokens.append(token)
+        return tokens
+
+    def segments_end_ts(self, res) -> List[float]:
+        return [segment["end"] for segment in res["segments"]]
+
+    def use_vad(self):
+        self.transcribe_kargs["vad"] = True
+
+    def set_translate_task(self):
+        self.transcribe_kargs["task"] = "translate"
+
+
+class FasterWhisperASR(ASRBase):
+    """Uses faster-whisper as the backend."""
+    sep = ""
+
+    def load_model(self, modelsize=None, cache_dir=None, model_dir=None):
+        from faster_whisper import WhisperModel
+
+        if model_dir is not None:
+            logger.debug(f"Loading whisper model from model_dir {model_dir}. "
+                         f"modelsize and cache_dir parameters are not used.")
+            model_size_or_path = model_dir
+        elif modelsize is not None:
+            model_size_or_path = modelsize
+        else:
+            raise ValueError("Either modelsize or model_dir must be set")
+        device = "cuda" if torch and torch.cuda.is_available() else "cpu"
+        compute_type = "float16" if device == "cuda" else "float32"
+
+        model = WhisperModel(
+            model_size_or_path,
+            device=device,
+            compute_type=compute_type,
+            download_root=cache_dir,
+        )
+        return model
+
+    def transcribe(self, audio: np.ndarray, init_prompt: str = "") -> list:
+        segments, info = self.model.transcribe(
+            audio,
+            language=self.original_language,
+            initial_prompt=init_prompt,
+            beam_size=5,
+            word_timestamps=True,
+            condition_on_previous_text=True,
+            **self.transcribe_kargs,
+        )
+        return list(segments)
+
+    def ts_words(self, segments) -> List[ASRToken]:
+        tokens = []
+        for segment in segments:
+            if segment.no_speech_prob > 0.9:
+                continue
+            for word in segment.words:
+                token = ASRToken(word.start, word.end, word.word, probability=word.probability)
+                tokens.append(token)
+        return tokens
+
+    def segments_end_ts(self, segments) -> List[float]:
+        return [segment.end for segment in segments]
+
+    def use_vad(self):
+        self.transcribe_kargs["vad_filter"] = True
+
+    def set_translate_task(self):
+        self.transcribe_kargs["task"] = "translate"
+
+
+class MLXWhisper(ASRBase):
+    """
+    Uses MLX Whisper optimized for Apple Silicon.
+    """
+    sep = ""
+
+    def load_model(self, modelsize=None, cache_dir=None, model_dir=None):
+        from mlx_whisper.transcribe import ModelHolder, transcribe
+        import mlx.core as mx
+
+        if model_dir is not None:
+            logger.debug(f"Loading whisper model from model_dir {model_dir}. modelsize parameter is not used.")
+            model_size_or_path = model_dir
+        elif modelsize is not None:
+            model_size_or_path = self.translate_model_name(modelsize)
+            logger.debug(f"Loading whisper model {modelsize}. You use mlx whisper, so {model_size_or_path} will be used.")
+        else:
+            raise ValueError("Either modelsize or model_dir must be set")
+
+        self.model_size_or_path = model_size_or_path
+        dtype = mx.float16
+        ModelHolder.get_model(model_size_or_path, dtype)
+        return transcribe
+
+    def translate_model_name(self, model_name):
+        model_mapping = {
+            "tiny.en": "mlx-community/whisper-tiny.en-mlx",
+            "tiny": "mlx-community/whisper-tiny-mlx",
+            "base.en": "mlx-community/whisper-base.en-mlx",
+            "base": "mlx-community/whisper-base-mlx",
+            "small.en": "mlx-community/whisper-small.en-mlx",
+            "small": "mlx-community/whisper-small-mlx",
+            "medium.en": "mlx-community/whisper-medium.en-mlx",
+            "medium": "mlx-community/whisper-medium-mlx",
+            "large-v1": "mlx-community/whisper-large-v1-mlx",
+            "large-v2": "mlx-community/whisper-large-v2-mlx",
+            "large-v3": "mlx-community/whisper-large-v3-mlx",
+            "large-v3-turbo": "mlx-community/whisper-large-v3-turbo",
+            "large": "mlx-community/whisper-large-mlx",
+        }
+        mlx_model_path = model_mapping.get(model_name)
+        if mlx_model_path:
+            return mlx_model_path
+        else:
+            raise ValueError(f"Model name '{model_name}' is not recognized or not supported.")
+
+    def transcribe(self, audio, init_prompt=""):
+        if self.transcribe_kargs:
+            logger.warning("Transcribe kwargs (vad, task) are not compatible with MLX Whisper and will be ignored.")
+        segments = self.model(
+            audio,
+            language=self.original_language,
+            initial_prompt=init_prompt,
+            word_timestamps=True,
+            condition_on_previous_text=True,
+            path_or_hf_repo=self.model_size_or_path,
+        )
+        return segments.get("segments", [])
+
+    def ts_words(self, segments) -> List[ASRToken]:
+        tokens = []
+        for segment in segments:
+            if segment.get("no_speech_prob", 0) > 0.9:
+                continue
+            for word in segment.get("words", []):
+                token = ASRToken(word["start"], word["end"], word["word"], probability=word["probability"])
+                tokens.append(token)
+        return tokens
+
+    def segments_end_ts(self, res) -> List[float]:
+        return [s["end"] for s in res]
+
+    def use_vad(self):
+        self.transcribe_kargs["vad_filter"] = True
+
+    def set_translate_task(self):
+        self.transcribe_kargs["task"] = "translate"
+
+
+class OpenaiApiASR(ASRBase):
+    """Uses OpenAI's Whisper API for transcription."""
+    def __init__(self, lan=None, temperature=0, logfile=sys.stderr):
+        self.logfile = logfile
+        self.modelname = "whisper-1"
+        self.original_language = None if lan == "auto" else lan
+        self.response_format = "verbose_json"
+        self.temperature = temperature
+        self.load_model()
+        self.use_vad_opt = False
+        self.task = "transcribe"
+
+    def load_model(self, *args, **kwargs):
+        from openai import OpenAI
+        self.client = OpenAI()
+        self.transcribed_seconds = 0
+
+    def ts_words(self, segments) -> List[ASRToken]:
+        """
+        Converts OpenAI API response words into ASRToken objects while
+        optionally skipping words that fall into no-speech segments.
+        """
+        no_speech_segments = []
+        if self.use_vad_opt:
+            for segment in segments.segments:
+                if segment.no_speech_prob > 0.8:
+                    no_speech_segments.append((segment.start, segment.end))
+        tokens = []
+        for word in segments.words:
+            start = word.start
+            end = word.end
+            if any(s[0] <= start <= s[1] for s in no_speech_segments):
+                continue
+            tokens.append(ASRToken(start, end, word.word))
+        return tokens
+
+    def segments_end_ts(self, res) -> List[float]:
+        return [s.end for s in res.words]
+
+    def transcribe(self, audio_data, prompt=None, *args, **kwargs):
+        buffer = io.BytesIO()
+        buffer.name = "temp.wav"
+        sf.write(buffer, audio_data, samplerate=16000, format="WAV", subtype="PCM_16")
+        buffer.seek(0)
+        self.transcribed_seconds += math.ceil(len(audio_data) / 16000)
+        params = {
+            "model": self.modelname,
+            "file": buffer,
+            "response_format": self.response_format,
+            "temperature": self.temperature,
+            "timestamp_granularities": ["word", "segment"],
+        }
+        if self.task != "translate" and self.original_language:
+            params["language"] = self.original_language
+        if prompt:
+            params["prompt"] = prompt
+        proc = self.client.audio.translations if self.task == "translate" else self.client.audio.transcriptions
+        transcript = proc.create(**params)
+        logger.debug(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds")
+        return transcript
+
+    def use_vad(self):
+        self.use_vad_opt = True
+
+    def set_translate_task(self):
+        self.task = "translate"

whisper_streaming_custom/online_asr.py

@@ -0,0 +1,453 @@
+import sys
+import numpy as np
+import logging
+from typing import List, Tuple, Optional
+from timed_objects import ASRToken, Sentence, Transcript
+
+logger = logging.getLogger(__name__)
+
+
+class HypothesisBuffer:
+    """
+    Buffer to store and process ASR hypothesis tokens.
+
+    It holds:
+      - committed_in_buffer: tokens that have been confirmed (committed)
+      - buffer: the last hypothesis that is not yet committed
+      - new: new tokens coming from the recognizer
+    """
+    def __init__(self, logfile=sys.stderr, confidence_validation=False):
+        self.confidence_validation = confidence_validation
+        self.committed_in_buffer: List[ASRToken] = []
+        self.buffer: List[ASRToken] = []
+        self.new: List[ASRToken] = []
+        self.last_committed_time = 0.0
+        self.last_committed_word: Optional[str] = None
+        self.logfile = logfile
+
+    def insert(self, new_tokens: List[ASRToken], offset: float):
+        """
+        Insert new tokens (after applying a time offset) and compare them with the 
+        already committed tokens. Only tokens that extend the committed hypothesis 
+        are added.
+        """
+        # Apply the offset to each token.
+        new_tokens = [token.with_offset(offset) for token in new_tokens]
+        # Only keep tokens that are roughly "new"
+        self.new = [token for token in new_tokens if token.start > self.last_committed_time - 0.1]
+
+        if self.new:
+            first_token = self.new[0]
+            if abs(first_token.start - self.last_committed_time) < 1:
+                if self.committed_in_buffer:
+                    committed_len = len(self.committed_in_buffer)
+                    new_len = len(self.new)
+                    # Try to match 1 to 5 consecutive tokens
+                    max_ngram = min(min(committed_len, new_len), 5)
+                    for i in range(1, max_ngram + 1):
+                        committed_ngram = " ".join(token.text for token in self.committed_in_buffer[-i:])
+                        new_ngram = " ".join(token.text for token in self.new[:i])
+                        if committed_ngram == new_ngram:
+                            removed = []
+                            for _ in range(i):
+                                removed_token = self.new.pop(0)
+                                removed.append(repr(removed_token))
+                            logger.debug(f"Removing last {i} words: {' '.join(removed)}")
+                            break
+
+    def flush(self) -> List[ASRToken]:
+        """
+        Returns the committed chunk, defined as the longest common prefix
+        between the previous hypothesis and the new tokens.
+        """
+        committed: List[ASRToken] = []
+        while self.new:
+            current_new = self.new[0]
+            if self.confidence_validation and current_new.probability and current_new.probability > 0.95:
+                committed.append(current_new)
+                self.last_committed_word = current_new.text
+                self.last_committed_time = current_new.end
+                self.new.pop(0)
+                self.buffer.pop(0) if self.buffer else None
+            elif not self.buffer:
+                break
+            elif current_new.text == self.buffer[0].text:
+                committed.append(current_new)
+                self.last_committed_word = current_new.text
+                self.last_committed_time = current_new.end
+                self.buffer.pop(0)
+                self.new.pop(0)
+            else:
+                break
+        self.buffer = self.new
+        self.new = []
+        self.committed_in_buffer.extend(committed)
+        return committed
+
+    def pop_committed(self, time: float):
+        """
+        Remove tokens (from the beginning) that have ended before `time`.
+        """
+        while self.committed_in_buffer and self.committed_in_buffer[0].end <= time:
+            self.committed_in_buffer.pop(0)
+
+
+
+class OnlineASRProcessor:
+    """
+    Processes incoming audio in a streaming fashion, calling the ASR system
+    periodically, and uses a hypothesis buffer to commit and trim recognized text.
+    
+    The processor supports two types of buffer trimming:
+      - "sentence": trims at sentence boundaries (using a sentence tokenizer)
+      - "segment": trims at fixed segment durations.
+    """
+    SAMPLING_RATE = 16000
+
+    def __init__(
+        self,
+        asr,
+        tokenize_method: Optional[callable] = None,
+        buffer_trimming: Tuple[str, float] = ("segment", 15),
+        confidence_validation = False,
+        logfile=sys.stderr,
+    ):
+        """
+        asr: An ASR system object (for example, a WhisperASR instance) that
+             provides a `transcribe` method, a `ts_words` method (to extract tokens),
+             a `segments_end_ts` method, and a separator attribute `sep`.
+        tokenize_method: A function that receives text and returns a list of sentence strings.
+        buffer_trimming: A tuple (option, seconds), where option is either "sentence" or "segment".
+        """
+        self.asr = asr
+        self.tokenize = tokenize_method
+        self.logfile = logfile
+        self.confidence_validation = confidence_validation
+        self.init()
+
+        self.buffer_trimming_way, self.buffer_trimming_sec = buffer_trimming
+
+        if self.buffer_trimming_way not in ["sentence", "segment"]:
+            raise ValueError("buffer_trimming must be either 'sentence' or 'segment'")
+        if self.buffer_trimming_sec <= 0:
+            raise ValueError("buffer_trimming_sec must be positive")
+        elif self.buffer_trimming_sec > 30:
+            logger.warning(
+                f"buffer_trimming_sec is set to {self.buffer_trimming_sec}, which is very long. It may cause OOM."
+            )
+
+    def init(self, offset: Optional[float] = None):
+        """Initialize or reset the processing buffers."""
+        self.audio_buffer = np.array([], dtype=np.float32)
+        self.transcript_buffer = HypothesisBuffer(logfile=self.logfile, confidence_validation=self.confidence_validation)
+        self.buffer_time_offset = offset if offset is not None else 0.0
+        self.transcript_buffer.last_committed_time = self.buffer_time_offset
+        self.committed: List[ASRToken] = []
+
+    def insert_audio_chunk(self, audio: np.ndarray):
+        """Append an audio chunk (a numpy array) to the current audio buffer."""
+        self.audio_buffer = np.append(self.audio_buffer, audio)
+
+    def prompt(self) -> Tuple[str, str]:
+        """
+        Returns a tuple: (prompt, context), where:
+          - prompt is a 200-character suffix of committed text that falls 
+            outside the current audio buffer.
+          - context is the committed text within the current audio buffer.
+        """
+        k = len(self.committed)
+        while k > 0 and self.committed[k - 1].end > self.buffer_time_offset:
+            k -= 1
+
+        prompt_tokens = self.committed[:k]
+        prompt_words = [token.text for token in prompt_tokens]
+        prompt_list = []
+        length_count = 0
+        # Use the last words until reaching 200 characters.
+        while prompt_words and length_count < 200:
+            word = prompt_words.pop(-1)
+            length_count += len(word) + 1
+            prompt_list.append(word)
+        non_prompt_tokens = self.committed[k:]
+        context_text = self.asr.sep.join(token.text for token in non_prompt_tokens)
+        return self.asr.sep.join(prompt_list[::-1]), context_text
+
+    def get_buffer(self):
+        """
+        Get the unvalidated buffer in string format.
+        """
+        return self.concatenate_tokens(self.transcript_buffer.buffer)
+        
+
+    def process_iter(self) -> Transcript:
+        """
+        Processes the current audio buffer.
+
+        Returns a Transcript object representing the committed transcript.
+        """
+        prompt_text, _ = self.prompt()
+        logger.debug(
+            f"Transcribing {len(self.audio_buffer)/self.SAMPLING_RATE:.2f} seconds from {self.buffer_time_offset:.2f}"
+        )
+        res = self.asr.transcribe(self.audio_buffer, init_prompt=prompt_text)
+        tokens = self.asr.ts_words(res)  # Expecting List[ASRToken]
+        self.transcript_buffer.insert(tokens, self.buffer_time_offset)
+        committed_tokens = self.transcript_buffer.flush()
+        self.committed.extend(committed_tokens)
+        completed = self.concatenate_tokens(committed_tokens)
+        logger.debug(f">>>> COMPLETE NOW: {completed.text}")
+        incomp = self.concatenate_tokens(self.transcript_buffer.buffer)
+        logger.debug(f"INCOMPLETE: {incomp.text}")
+
+        if committed_tokens and self.buffer_trimming_way == "sentence":
+            if len(self.audio_buffer) / self.SAMPLING_RATE > self.buffer_trimming_sec:
+                self.chunk_completed_sentence()
+
+        s = self.buffer_trimming_sec if self.buffer_trimming_way == "segment" else 30
+        if len(self.audio_buffer) / self.SAMPLING_RATE > s:
+            self.chunk_completed_segment(res)
+            logger.debug("Chunking segment")
+        logger.debug(
+            f"Length of audio buffer now: {len(self.audio_buffer)/self.SAMPLING_RATE:.2f} seconds"
+        )
+        return committed_tokens
+
+    def chunk_completed_sentence(self):
+        """
+        If the committed tokens form at least two sentences, chunk the audio
+        buffer at the end time of the penultimate sentence.
+        """
+        if not self.committed:
+            return
+        logger.debug("COMPLETED SENTENCE: " + " ".join(token.text for token in self.committed))
+        sentences = self.words_to_sentences(self.committed)
+        for sentence in sentences:
+            logger.debug(f"\tSentence: {sentence.text}")
+        if len(sentences) < 2:
+            return
+        # Keep the last two sentences.
+        while len(sentences) > 2:
+            sentences.pop(0)
+        chunk_time = sentences[-2].end
+        logger.debug(f"--- Sentence chunked at {chunk_time:.2f}")
+        self.chunk_at(chunk_time)
+
+    def chunk_completed_segment(self, res):
+        """
+        Chunk the audio buffer based on segment-end timestamps reported by the ASR.
+        """
+        if not self.committed:
+            return
+        ends = self.asr.segments_end_ts(res)
+        last_committed_time = self.committed[-1].end
+        if len(ends) > 1:
+            e = ends[-2] + self.buffer_time_offset
+            while len(ends) > 2 and e > last_committed_time:
+                ends.pop(-1)
+                e = ends[-2] + self.buffer_time_offset
+            if e <= last_committed_time:
+                logger.debug(f"--- Segment chunked at {e:.2f}")
+                self.chunk_at(e)
+            else:
+                logger.debug("--- Last segment not within committed area")
+        else:
+            logger.debug("--- Not enough segments to chunk")
+
+    def chunk_at(self, time: float):
+        """
+        Trim both the hypothesis and audio buffer at the given time.
+        """
+        logger.debug(f"Chunking at {time:.2f}s")
+        logger.debug(
+            f"Audio buffer length before chunking: {len(self.audio_buffer)/self.SAMPLING_RATE:.2f}s"
+        )
+        self.transcript_buffer.pop_committed(time)
+        cut_seconds = time - self.buffer_time_offset
+        self.audio_buffer = self.audio_buffer[int(cut_seconds * self.SAMPLING_RATE):]
+        self.buffer_time_offset = time
+        logger.debug(
+            f"Audio buffer length after chunking: {len(self.audio_buffer)/self.SAMPLING_RATE:.2f}s"
+        )
+
+    def words_to_sentences(self, tokens: List[ASRToken]) -> List[Sentence]:
+        """
+        Converts a list of tokens to a list of Sentence objects using the provided
+        sentence tokenizer.
+        """
+        if not tokens:
+            return []
+
+        full_text = " ".join(token.text for token in tokens)
+
+        if self.tokenize:
+            try:
+                sentence_texts = self.tokenize(full_text)
+            except Exception as e:
+                # Some tokenizers (e.g., MosesSentenceSplitter) expect a list input.
+                try:
+                    sentence_texts = self.tokenize([full_text])
+                except Exception as e2:
+                    raise ValueError("Tokenization failed") from e2
+        else:
+            sentence_texts = [full_text]
+
+        sentences: List[Sentence] = []
+        token_index = 0
+        for sent_text in sentence_texts:
+            sent_text = sent_text.strip()
+            if not sent_text:
+                continue
+            sent_tokens = []
+            accumulated = ""
+            # Accumulate tokens until roughly matching the length of the sentence text.
+            while token_index < len(tokens) and len(accumulated) < len(sent_text):
+                token = tokens[token_index]
+                accumulated = (accumulated + " " + token.text).strip() if accumulated else token.text
+                sent_tokens.append(token)
+                token_index += 1
+            if sent_tokens:
+                sentence = Sentence(
+                    start=sent_tokens[0].start,
+                    end=sent_tokens[-1].end,
+                    text=" ".join(t.text for t in sent_tokens),
+                )
+                sentences.append(sentence)
+        return sentences
+    def finish(self) -> Transcript:
+        """
+        Flush the remaining transcript when processing ends.
+        """
+        remaining_tokens = self.transcript_buffer.buffer
+        final_transcript = self.concatenate_tokens(remaining_tokens)
+        logger.debug(f"Final non-committed transcript: {final_transcript}")
+        self.buffer_time_offset += len(self.audio_buffer) / self.SAMPLING_RATE
+        return final_transcript
+
+    def concatenate_tokens(
+        self,
+        tokens: List[ASRToken],
+        sep: Optional[str] = None,
+        offset: float = 0
+    ) -> Transcript:
+        sep = sep if sep is not None else self.asr.sep
+        text = sep.join(token.text for token in tokens)
+        probability = sum(token.probability for token in tokens if token.probability) / len(tokens) if tokens else None
+        if tokens:
+            start = offset + tokens[0].start
+            end = offset + tokens[-1].end
+        else:
+            start = None
+            end = None
+        return Transcript(start, end, text, probability=probability)
+
+
+class VACOnlineASRProcessor:
+    """
+    Wraps an OnlineASRProcessor with a Voice Activity Controller (VAC).
+    
+    It receives small chunks of audio, applies VAD (e.g. with Silero),
+    and when the system detects a pause in speech (or end of an utterance)
+    it finalizes the utterance immediately.
+    """
+    SAMPLING_RATE = 16000
+
+    def __init__(self, online_chunk_size: float, *args, **kwargs):
+        self.online_chunk_size = online_chunk_size
+        self.online = OnlineASRProcessor(*args, **kwargs)
+
+        # Load a VAD model (e.g. Silero VAD)
+        import torch
+        model, _ = torch.hub.load(repo_or_dir="snakers4/silero-vad", model="silero_vad")
+        from silero_vad_iterator import FixedVADIterator
+
+        self.vac = FixedVADIterator(model)
+        self.logfile = self.online.logfile
+        self.init()
+
+    def init(self):
+        self.online.init()
+        self.vac.reset_states()
+        self.current_online_chunk_buffer_size = 0
+        self.is_currently_final = False
+        self.status: Optional[str] = None  # "voice" or "nonvoice"
+        self.audio_buffer = np.array([], dtype=np.float32)
+        self.buffer_offset = 0  # in frames
+
+    def clear_buffer(self):
+        self.buffer_offset += len(self.audio_buffer)
+        self.audio_buffer = np.array([], dtype=np.float32)
+
+    def insert_audio_chunk(self, audio: np.ndarray):
+        """
+        Process an incoming small audio chunk:
+          - run VAD on the chunk,
+          - decide whether to send the audio to the online ASR processor immediately,
+          - and/or to mark the current utterance as finished.
+        """
+        res = self.vac(audio)
+        self.audio_buffer = np.append(self.audio_buffer, audio)
+
+        if res is not None:
+            # VAD returned a result; adjust the frame number
+            frame = list(res.values())[0] - self.buffer_offset
+            if "start" in res and "end" not in res:
+                self.status = "voice"
+                send_audio = self.audio_buffer[frame:]
+                self.online.init(offset=(frame + self.buffer_offset) / self.SAMPLING_RATE)
+                self.online.insert_audio_chunk(send_audio)
+                self.current_online_chunk_buffer_size += len(send_audio)
+                self.clear_buffer()
+            elif "end" in res and "start" not in res:
+                self.status = "nonvoice"
+                send_audio = self.audio_buffer[:frame]
+                self.online.insert_audio_chunk(send_audio)
+                self.current_online_chunk_buffer_size += len(send_audio)
+                self.is_currently_final = True
+                self.clear_buffer()
+            else:
+                beg = res["start"] - self.buffer_offset
+                end = res["end"] - self.buffer_offset
+                self.status = "nonvoice"
+                send_audio = self.audio_buffer[beg:end]
+                self.online.init(offset=(beg + self.buffer_offset) / self.SAMPLING_RATE)
+                self.online.insert_audio_chunk(send_audio)
+                self.current_online_chunk_buffer_size += len(send_audio)
+                self.is_currently_final = True
+                self.clear_buffer()
+        else:
+            if self.status == "voice":
+                self.online.insert_audio_chunk(self.audio_buffer)
+                self.current_online_chunk_buffer_size += len(self.audio_buffer)
+                self.clear_buffer()
+            else:
+                # Keep 1 second worth of audio in case VAD later detects voice,
+                # but trim to avoid unbounded memory usage.
+                self.buffer_offset += max(0, len(self.audio_buffer) - self.SAMPLING_RATE)
+                self.audio_buffer = self.audio_buffer[-self.SAMPLING_RATE:]
+
+    def process_iter(self) -> Transcript:
+        """
+        Depending on the VAD status and the amount of accumulated audio,
+        process the current audio chunk.
+        """
+        if self.is_currently_final:
+            return self.finish()
+        elif self.current_online_chunk_buffer_size > self.SAMPLING_RATE * self.online_chunk_size:
+            self.current_online_chunk_buffer_size = 0
+            return self.online.process_iter()
+        else:
+            logger.debug("No online update, only VAD")
+            return Transcript(None, None, "")
+
+    def finish(self) -> Transcript:
+        """Finish processing by flushing any remaining text."""
+        result = self.online.finish()
+        self.current_online_chunk_buffer_size = 0
+        self.is_currently_final = False
+        return result
+    
+    def get_buffer(self):
+        """
+        Get the unvalidated buffer in string format.
+        """
+        return self.online.concatenate_tokens(self.online.transcript_buffer.buffer).text

whisper_streaming_custom/whisper_online.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+import sys
+import numpy as np
+import librosa
+from functools import lru_cache
+import time
+import logging
+from .backends import FasterWhisperASR, MLXWhisper, WhisperTimestampedASR, OpenaiApiASR
+from .online_asr import OnlineASRProcessor, VACOnlineASRProcessor
+
+logger = logging.getLogger(__name__)
+
+
+
+WHISPER_LANG_CODES = "af,am,ar,as,az,ba,be,bg,bn,bo,br,bs,ca,cs,cy,da,de,el,en,es,et,eu,fa,fi,fo,fr,gl,gu,ha,haw,he,hi,hr,ht,hu,hy,id,is,it,ja,jw,ka,kk,km,kn,ko,la,lb,ln,lo,lt,lv,mg,mi,mk,ml,mn,mr,ms,mt,my,ne,nl,nn,no,oc,pa,pl,ps,pt,ro,ru,sa,sd,si,sk,sl,sn,so,sq,sr,su,sv,sw,ta,te,tg,th,tk,tl,tr,tt,uk,ur,uz,vi,yi,yo,zh".split(
+    ","
+)
+
+
+def create_tokenizer(lan):
+    """returns an object that has split function that works like the one of MosesTokenizer"""
+
+    assert (
+        lan in WHISPER_LANG_CODES
+    ), "language must be Whisper's supported lang code: " + " ".join(WHISPER_LANG_CODES)
+
+    if lan == "uk":
+        import tokenize_uk
+
+        class UkrainianTokenizer:
+            def split(self, text):
+                return tokenize_uk.tokenize_sents(text)
+
+        return UkrainianTokenizer()
+
+    # supported by fast-mosestokenizer
+    if (
+        lan
+        in "as bn ca cs de el en es et fi fr ga gu hi hu is it kn lt lv ml mni mr nl or pa pl pt ro ru sk sl sv ta te yue zh".split()
+    ):
+        from mosestokenizer import MosesSentenceSplitter        
+
+        return MosesSentenceSplitter(lan)
+
+    # the following languages are in Whisper, but not in wtpsplit:
+    if (
+        lan
+        in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split()
+    ):
+        logger.debug(
+            f"{lan} code is not supported by wtpsplit. Going to use None lang_code option."
+        )
+        lan = None
+
+    from wtpsplit import WtP
+
+    # downloads the model from huggingface on the first use
+    wtp = WtP("wtp-canine-s-12l-no-adapters")
+
+    class WtPtok:
+        def split(self, sent):
+            return wtp.split(sent, lang_code=lan)
+
+    return WtPtok()
+
+
+def backend_factory(args):
+    backend = args.backend
+    if backend == "openai-api":
+        logger.debug("Using OpenAI API.")
+        asr = OpenaiApiASR(lan=args.lan)
+    else:
+        if backend == "faster-whisper":
+            asr_cls = FasterWhisperASR
+        elif backend == "mlx-whisper":
+            asr_cls = MLXWhisper
+        else:
+            asr_cls = WhisperTimestampedASR
+
+        # Only for FasterWhisperASR and WhisperTimestampedASR
+        size = args.model
+        t = time.time()
+        logger.info(f"Loading Whisper {size} model for language {args.lan}...")
+        asr = asr_cls(
+            modelsize=size,
+            lan=args.lan,
+            cache_dir=args.model_cache_dir,
+            model_dir=args.model_dir,
+        )
+        e = time.time()
+        logger.info(f"done. It took {round(e-t,2)} seconds.")
+
+    # Apply common configurations
+    if getattr(args, "vad", False):  # Checks if VAD argument is present and True
+        logger.info("Setting VAD filter")
+        asr.use_vad()
+
+    language = args.lan
+    if args.task == "translate":
+        asr.set_translate_task()
+        tgt_language = "en"  # Whisper translates into English
+    else:
+        tgt_language = language  # Whisper transcribes in this language
+
+    # Create the tokenizer
+    if args.buffer_trimming == "sentence":
+
+        tokenizer = create_tokenizer(tgt_language)
+    else:
+        tokenizer = None
+    return asr, tokenizer
+
+def online_factory(args, asr, tokenizer, logfile=sys.stderr):
+    if args.vac:
+        online = VACOnlineASRProcessor(
+            args.min_chunk_size,
+            asr,
+            tokenizer,
+            logfile=logfile,
+            buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec),
+            confidence_validation = args.confidence_validation
+        )
+    else:
+        online = OnlineASRProcessor(
+            asr,
+            tokenizer,
+            logfile=logfile,
+            buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec),
+            confidence_validation = args.confidence_validation
+        )
+    return online
+  
+def asr_factory(args, logfile=sys.stderr):
+    """
+    Creates and configures an ASR and ASR Online instance based on the specified backend and arguments.
+    """
+    asr, tokenizer = backend_factory(args)
+    online = online_factory(args, asr, tokenizer, logfile=logfile)
+    return asr, online
+
+def warmup_asr(asr, warmup_file=None, timeout=5):
+    """
+    Warmup the ASR model by transcribing a short audio file.
+    """
+    import os
+    import tempfile
+    
+    
+    if warmup_file is None:
+        # Download JFK sample if not already present
+        jfk_url = "https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav"
+        temp_dir = tempfile.gettempdir()
+        warmup_file = os.path.join(temp_dir, "whisper_warmup_jfk.wav")
+        
+        if not os.path.exists(warmup_file):
+            logger.debug(f"Downloading warmup file from {jfk_url}")
+            print(f"Downloading warmup file from {jfk_url}")
+            import time
+            import urllib.request
+            import urllib.error
+            import socket
+            
+            original_timeout = socket.getdefaulttimeout()
+            socket.setdefaulttimeout(timeout)
+            
+            start_time = time.time()
+            try:
+                urllib.request.urlretrieve(jfk_url, warmup_file)
+                logger.debug(f"Download successful in {time.time() - start_time:.2f}s")
+            except (urllib.error.URLError, socket.timeout) as e:
+                logger.warning(f"Download failed: {e}. Proceeding without warmup.")
+                return False
+            finally:
+                socket.setdefaulttimeout(original_timeout)
+    elif not warmup_file:
+        return False 
+    
+    if not warmup_file or not os.path.exists(warmup_file) or os.path.getsize(warmup_file) == 0:
+        logger.warning(f"Warmup file {warmup_file} invalid or missing.")
+        return False
+    
+    print(f"Warmping up Whisper with {warmup_file}")
+    try:
+        import librosa
+        audio, sr = librosa.load(warmup_file, sr=16000)
+    except Exception as e:
+        logger.warning(f"Failed to load audio file: {e}")
+        return False
+            
+    # Process the audio
+    asr.transcribe(audio)
+
+    logger.info("Whisper is warmed up")
+