Merge commit '2d0e88443951d8635ba0d50f005a9e9c30aaee05' as 'libs/whisper_streaming'

libs/whisper_streaming/.gitignore

@@ -0,0 +1,129 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

libs/whisper_streaming/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 ÚFAL
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

libs/whisper_streaming/README.md

@@ -0,0 +1,263 @@
+# whisper_streaming
+Whisper realtime streaming for long speech-to-text transcription and translation
+
+**Turning Whisper into Real-Time Transcription System**
+
+Demonstration paper, by [Dominik Macháček](https://ufal.mff.cuni.cz/dominik-machacek), [Raj Dabre](https://prajdabre.github.io/), [Ondřej Bojar](https://ufal.mff.cuni.cz/ondrej-bojar), 2023
+
+Abstract:    Whisper is one of the recent state-of-the-art multilingual speech recognition and translation models, however, it is not designed for real-time transcription. In this paper, we build on top of Whisper and create Whisper-Streaming, an implementation of real-time speech transcription and translation of Whisper-like models. Whisper-Streaming uses local agreement policy with self-adaptive latency to enable streaming transcription. We show that Whisper-Streaming achieves high quality and 3.3 seconds latency on unsegmented long-form speech transcription test set, and we demonstrate its robustness and practical usability as a component in live transcription service at a multilingual conference. 
+
+
+[Paper PDF](https://aclanthology.org/2023.ijcnlp-demo.3.pdf), [Demo video](https://player.vimeo.com/video/840442741)
+
+[Slides](http://ufallab.ms.mff.cuni.cz/~machacek/pre-prints/AACL23-2.11.2023-Turning-Whisper-oral.pdf) -- 15 minutes oral presentation at IJCNLP-AACL 2023
+
+Please, cite us. [ACL Anthology](https://aclanthology.org/2023.ijcnlp-demo.3/), [Bibtex citation](https://aclanthology.org/2023.ijcnlp-demo.3.bib):
+
+```
+@inproceedings{machacek-etal-2023-turning,
+    title = "Turning Whisper into Real-Time Transcription System",
+    author = "Mach{\'a}{\v{c}}ek, Dominik  and
+      Dabre, Raj  and
+      Bojar, Ond{\v{r}}ej",
+    editor = "Saha, Sriparna  and
+      Sujaini, Herry",
+    booktitle = "Proceedings of the 13th International Joint Conference on Natural Language Processing and the 3rd Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics: System Demonstrations",
+    month = nov,
+    year = "2023",
+    address = "Bali, Indonesia",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2023.ijcnlp-demo.3",
+    pages = "17--24",
+}
+```
+
+## Installation
+
+1) ``pip install librosa soundfile`` -- audio processing library
+
+2) Whisper backend.
+
+ Several alternative backends are integrated. The most recommended one is [faster-whisper](https://github.com/guillaumekln/faster-whisper) with GPU support. Follow their instructions for NVIDIA libraries -- we succeeded with CUDNN 8.5.0 and CUDA 11.7. Install with `pip install faster-whisper`.
+
+Alternative, less restrictive, but slower backend is [whisper-timestamped](https://github.com/linto-ai/whisper-timestamped): `pip install git+https://github.com/linto-ai/whisper-timestamped`
+
+Thirdly, it's also possible to run this software from the [OpenAI Whisper API](https://platform.openai.com/docs/api-reference/audio/createTranscription). This solution is fast and requires no GPU, just a small VM will suffice, but you will need to pay OpenAI for api access. Also note that, since each audio fragment is processed multiple times, the [price](https://openai.com/pricing) will be higher than obvious from the pricing page, so keep an eye on costs while using. Setting a higher chunk-size will reduce costs significantly. 
+Install with: `pip install openai`
+
+For running with the openai-api backend, make sure that your [OpenAI api key](https://platform.openai.com/api-keys) is set in the `OPENAI_API_KEY` environment variable. For example, before running, do: `export OPENAI_API_KEY=sk-xxx` with *sk-xxx* replaced with your api key. 
+
+The backend is loaded only when chosen. The unused one does not have to be installed.
+
+3) For voice activity controller: `pip install torch torchaudio`. Optional, but very recommended.
+
+<details>
+<summary>4) Optional, not recommended: sentence segmenter (aka sentence tokenizer)</summary>
+
+Two buffer trimming options are integrated and evaluated. They have impact on
+the quality and latency. The default "segment" option performs better according
+to our tests and does not require any sentence segmentation installed. 
+
+The other option, "sentence" -- trimming at the end of confirmed sentences,
+requires sentence segmenter installed.  It splits punctuated text to sentences by full
+stops, avoiding the dots that are not full stops. The segmenters are language
+specific.  The unused one does not have to be installed. We integrate the
+following segmenters, but suggestions for better alternatives are welcome.
+
+- `pip install opus-fast-mosestokenizer` for the languages with codes `as bn ca cs de el en es et fi fr ga gu hi hu is it kn lt lv ml mni mr nl or pa pl pt ro ru sk sl sv ta te yue zh`
+
+- `pip install tokenize_uk` for Ukrainian -- `uk`
+
+- for other languages, we integrate a good performing multi-lingual model of `wtpslit`. It requires `pip install torch wtpsplit`, and its neural model `wtp-canine-s-12l-no-adapters`. It is downloaded to the default huggingface cache during the first use. 
+
+- we did not find a segmenter for languages `as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt` that are supported by Whisper and not by wtpsplit. The default fallback option for them is wtpsplit with unspecified language. Alternative suggestions welcome.
+
+In case of installation issues of opus-fast-mosestokenizer, especially on Windows and Mac, we recommend using only the "segment" option that does not require it.
+</details>
+
+## Usage
+
+### Real-time simulation from audio file
+
+```
+whisper_online.py -h
+usage: whisper_online.py [-h] [--min-chunk-size MIN_CHUNK_SIZE] [--model {tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large}] [--model_cache_dir MODEL_CACHE_DIR]
+                         [--model_dir MODEL_DIR] [--lan LAN] [--task {transcribe,translate}] [--backend {faster-whisper,whisper_timestamped,openai-api}] [--vac] [--vac-chunk-size VAC_CHUNK_SIZE] [--vad]
+                         [--buffer_trimming {sentence,segment}] [--buffer_trimming_sec BUFFER_TRIMMING_SEC] [-l {DEBUG,INFO,WARNING,ERROR,CRITICAL}] [--start_at START_AT] [--offline] [--comp_unaware]
+                         audio_path
+
+positional arguments:
+  audio_path            Filename of 16kHz mono channel wav, on which live streaming is simulated.
+
+options:
+  -h, --help            show this help message and exit
+  --min-chunk-size MIN_CHUNK_SIZE
+                        Minimum audio chunk size in seconds. It waits up to this time to do processing. If the processing takes shorter time, it waits, otherwise it processes the whole segment that was
+                        received by this time.
+  --model {tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large}
+                        Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.
+  --model_cache_dir MODEL_CACHE_DIR
+                        Overriding the default model cache dir where models downloaded from the hub are saved
+  --model_dir MODEL_DIR
+                        Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.
+  --lan LAN, --language LAN
+                        Source language code, e.g. en,de,cs, or 'auto' for language detection.
+  --task {transcribe,translate}
+                        Transcribe or translate.
+  --backend {faster-whisper,whisper_timestamped,openai-api}
+                        Load only this backend for Whisper processing.
+  --vac                 Use VAC = voice activity controller. Recommended. Requires torch.
+  --vac-chunk-size VAC_CHUNK_SIZE
+                        VAC sample size in seconds.
+  --vad                 Use VAD = voice activity detection, with the default parameters.
+  --buffer_trimming {sentence,segment}
+                        Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter
+                        must be installed for "sentence" option.
+  --buffer_trimming_sec BUFFER_TRIMMING_SEC
+                        Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.
+  -l {DEBUG,INFO,WARNING,ERROR,CRITICAL}, --log-level {DEBUG,INFO,WARNING,ERROR,CRITICAL}
+                        Set the log level
+  --start_at START_AT   Start processing audio at this time.
+  --offline             Offline mode.
+  --comp_unaware        Computationally unaware simulation.
+```
+
+Example:
+
+It simulates realtime processing from a pre-recorded mono 16k wav file.
+
+```
+python3 whisper_online.py en-demo16.wav --language en --min-chunk-size 1 > out.txt
+```
+
+Simulation modes:
+
+- default mode, no special option: real-time simulation from file, computationally aware. The chunk size is `MIN_CHUNK_SIZE` or larger, if more audio arrived during last update computation.
+
+- `--comp_unaware` option: computationally unaware simulation. It means that the timer that counts the emission times "stops" when the model is computing. The chunk size is always `MIN_CHUNK_SIZE`. The latency is caused only by the model being unable to confirm the output, e.g. because of language ambiguity etc., and not because of slow hardware or suboptimal implementation. We implement this feature for finding the lower bound for latency.
+
+- `--start_at START_AT`: Start processing audio at this time. The first update receives the whole audio by `START_AT`. It is useful for debugging, e.g. when we observe a bug in a specific time in audio file, and want to reproduce it quickly, without long waiting.
+
+- `--offline` option: It processes the whole audio file at once, in offline mode. We implement it to find out the lowest possible WER on given audio file.
+
+
+
+### Output format
+
+```
+2691.4399 300 1380 Chairman, thank you.
+6914.5501 1940 4940 If the debate today had a
+9019.0277 5160 7160 the subject the situation in
+10065.1274 7180 7480 Gaza
+11058.3558 7480 9460 Strip, I might
+12224.3731 9460 9760 have
+13555.1929 9760 11060 joined Mrs.
+14928.5479 11140 12240 De Kaiser and all the
+16588.0787 12240 12560 other
+18324.9285 12560 14420 colleagues across the
+```
+
+[See description here](https://github.com/ufal/whisper_streaming/blob/d915d790a62d7be4e7392dde1480e7981eb142ae/whisper_online.py#L361)
+
+### As a module
+
+TL;DR: use OnlineASRProcessor object and its methods insert_audio_chunk and process_iter. 
+
+The code whisper_online.py is nicely commented, read it as the full documentation.
+
+
+This pseudocode describes the interface that we suggest for your implementation. You can implement any features that you need for your application.
+
+```python
+from whisper_online import *
+
+src_lan = "en"  # source language
+tgt_lan = "en"  # target language  -- same as source for ASR, "en" if translate task is used
+
+asr = FasterWhisperASR(lan, "large-v2")  # loads and wraps Whisper model
+# set options:
+# asr.set_translate_task()  # it will translate from lan into English
+# asr.use_vad()  # set using VAD
+
+online = OnlineASRProcessor(asr)  # create processing object with default buffer trimming option
+
+while audio_has_not_ended:   # processing loop:
+	a = # receive new audio chunk (and e.g. wait for min_chunk_size seconds first, ...)
+	online.insert_audio_chunk(a)
+	o = online.process_iter()
+	print(o) # do something with current partial output
+# at the end of this audio processing
+o = online.finish()
+print(o)  # do something with the last output
+
+
+online.init()  # refresh if you're going to re-use the object for the next audio
+```
+
+### Server -- real-time from mic
+
+`whisper_online_server.py` has the same model options as `whisper_online.py`, plus `--host` and `--port` of the TCP connection and the `--warmup-file`. See the help message (`-h` option).
+
+Client example:
+
+```
+arecord -f S16_LE -c1 -r 16000 -t raw -D default | nc localhost 43001
+```
+
+- arecord sends realtime audio from a sound device (e.g. mic), in raw audio format -- 16000 sampling rate, mono channel, S16\_LE -- signed 16-bit integer low endian. (use the alternative to arecord that works for you)
+
+- nc is netcat with server's host and port
+
+
+## Background
+
+Default Whisper is intended for audio chunks of at most 30 seconds that contain
+one full sentence. Longer audio files must be split to shorter chunks and
+merged with "init prompt". In low latency simultaneous streaming mode, the
+simple and naive chunking fixed-sized windows does not work well, it can split
+a word in the middle. It is also necessary to know when the transcribt is
+stable, should be confirmed ("commited") and followed up, and when the future
+content makes the transcript clearer. 
+
+For that, there is LocalAgreement-n policy: if n consecutive updates, each with
+a newly available audio stream chunk, agree on a prefix transcript, it is
+confirmed. (Reference: CUNI-KIT at IWSLT 2022 etc.)
+
+In this project, we re-use the idea of Peter Polák from this demo:
+https://github.com/pe-trik/transformers/blob/online_decode/examples/pytorch/online-decoding/whisper-online-demo.py
+However, it doesn't do any sentence segmentation, but Whisper produces
+punctuation and the libraries `faster-whisper` and `whisper_transcribed` make
+word-level timestamps. In short: we
+consecutively process new audio chunks, emit the transcripts that are confirmed
+by 2 iterations, and scroll the audio processing buffer on a timestamp of a
+confirmed complete sentence. The processing audio buffer is not too long and
+the processing is fast.
+
+In more detail: we use the init prompt, we handle the inaccurate timestamps, we
+re-process confirmed sentence prefixes and skip them, making sure they don't
+overlap, and we limit the processing buffer window. 
+
+### Performance evaluation
+
+[See the paper.](http://www.afnlp.org/conferences/ijcnlp2023/proceedings/main-demo/cdrom/pdf/2023.ijcnlp-demo.3.pdf)
+
+### Contributions
+
+Contributions are welcome. We acknowledge especially:
+
+- [The GitHub contributors](https://github.com/ufal/whisper_streaming/graphs/contributors) for their pull requests with new features and bugfixes.
+- [Nice explanation video](https://www.youtube.com/watch?v=_spinzpEeFM) -- published on 31st March 2024, note that newer updates are not included.
+- [The translation of this repo into Chinese.](https://github.com/Gloridust/whisper_streaming_CN)
+- [Ondřej Plátek](https://opla.cz/) for the paper pre-review.
+
+Credits:
+
+- [Peter Polák](https://ufal.mff.cuni.cz/peter-polak) for the original idea.
+- The UEDIN team of the [ELITR project](https://elitr.eu) for the original line_packet.py.
+- Silero Team for their VAD [model](https://github.com/snakers4/silero-vad) and [VADIterator](https://github.com/ufal/whisper_streaming/blob/47caa80588ee9c0fa8945a5d05f0aea6315eb837/silero_vad.py#L8).
+
+
+## Contact
+
+Dominik Macháček, machacek@ufal.mff.cuni.cz
+

libs/whisper_streaming/line_packet.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+"""Functions for sending and receiving individual lines of text over a socket.
+
+A line is transmitted using one or more fixed-size packets of UTF-8 bytes
+containing:
+
+  - Zero or more bytes of UTF-8, excluding \n and \0, followed by
+
+  - Zero or more \0 bytes as required to pad the packet to PACKET_SIZE
+
+Originally from the UEDIN team of the ELITR project. 
+"""
+
+PACKET_SIZE = 65536
+
+
+def send_one_line(socket, text, pad_zeros=False):
+    """Sends a line of text over the given socket.
+
+    The 'text' argument should contain a single line of text (line break
+    characters are optional). Line boundaries are determined by Python's
+    str.splitlines() function [1]. We also count '\0' as a line terminator.
+    If 'text' contains multiple lines then only the first will be sent.
+
+    If the send fails then an exception will be raised.
+
+    [1] https://docs.python.org/3.5/library/stdtypes.html#str.splitlines
+
+    Args:
+        socket: a socket object.
+        text: string containing a line of text for transmission.
+    """
+    text.replace('\0', '\n')
+    lines = text.splitlines()
+    first_line = '' if len(lines) == 0 else lines[0]
+    # TODO Is there a better way of handling bad input than 'replace'?
+    data = first_line.encode('utf-8', errors='replace') + b'\n' + (b'\0' if pad_zeros else b'')
+    for offset in range(0, len(data), PACKET_SIZE):
+        bytes_remaining = len(data) - offset
+        if bytes_remaining < PACKET_SIZE:
+            padding_length = PACKET_SIZE - bytes_remaining
+            packet = data[offset:] + (b'\0' * padding_length if pad_zeros else b'')
+        else:
+            packet = data[offset:offset+PACKET_SIZE]
+        socket.sendall(packet)
+
+
+def receive_one_line(socket):
+    """Receives a line of text from the given socket.
+
+    This function will (attempt to) receive a single line of text. If data is
+    currently unavailable then it will block until data becomes available or
+    the sender has closed the connection (in which case it will return an
+    empty string).
+
+    The string should not contain any newline characters, but if it does then
+    only the first line will be returned.
+
+    Args:
+        socket: a socket object.
+
+    Returns:
+        A string representing a single line with a terminating newline or
+        None if the connection has been closed.
+    """
+    data = b''
+    while True:
+        packet = socket.recv(PACKET_SIZE)
+        if not packet:  # Connection has been closed.
+            return None
+        data += packet
+        if b'\0' in packet:
+            break
+    # TODO Is there a better way of handling bad input than 'replace'?
+    text = data.decode('utf-8', errors='replace').strip('\0')
+    lines = text.split('\n')
+    return lines[0] + '\n'
+
+
+def receive_lines(socket):
+    try:
+        data = socket.recv(PACKET_SIZE)
+    except BlockingIOError:
+        return []
+    if data is None:  # Connection has been closed.
+        return None
+    # TODO Is there a better way of handling bad input than 'replace'?
+    text = data.decode('utf-8', errors='replace').strip('\0')
+    lines = text.split('\n')
+    if len(lines)==1 and not lines[0]:
+        return None
+    return lines

libs/whisper_streaming/silero_vad.py

@@ -0,0 +1,97 @@
+import torch
+
+# This is copied from silero-vad's vad_utils.py:
+# https://github.com/snakers4/silero-vad/blob/f6b1294cb27590fb2452899df98fb234dfef1134/utils_vad.py#L340
+
+# Their licence is MIT, same as ours: https://github.com/snakers4/silero-vad/blob/f6b1294cb27590fb2452899df98fb234dfef1134/LICENSE
+
+class VADIterator:
+    def __init__(self,
+                 model,
+                 threshold: float = 0.5,
+                 sampling_rate: int = 16000,
+                 min_silence_duration_ms: int = 100,
+                 speech_pad_ms: int = 30
+                 ):
+
+        """
+        Class for stream imitation
+
+        Parameters
+        ----------
+        model: preloaded .jit silero VAD model
+
+        threshold: float (default - 0.5)
+            Speech threshold. Silero VAD outputs speech probabilities for each audio chunk, probabilities ABOVE this value are considered as SPEECH.
+            It is better to tune this parameter for each dataset separately, but "lazy" 0.5 is pretty good for most datasets.
+
+        sampling_rate: int (default - 16000)
+            Currently silero VAD models support 8000 and 16000 sample rates
+
+        min_silence_duration_ms: int (default - 100 milliseconds)
+            In the end of each speech chunk wait for min_silence_duration_ms before separating it
+
+        speech_pad_ms: int (default - 30 milliseconds)
+            Final speech chunks are padded by speech_pad_ms each side
+        """
+
+        self.model = model
+        self.threshold = threshold
+        self.sampling_rate = sampling_rate
+
+        if sampling_rate not in [8000, 16000]:
+            raise ValueError('VADIterator does not support sampling rates other than [8000, 16000]')
+
+        self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000
+        self.speech_pad_samples = sampling_rate * speech_pad_ms / 1000
+        self.reset_states()
+
+    def reset_states(self):
+
+        self.model.reset_states()
+        self.triggered = False
+        self.temp_end = 0
+        self.current_sample = 0
+
+    def __call__(self, x, return_seconds=False):
+        """
+        x: torch.Tensor
+            audio chunk (see examples in repo)
+
+        return_seconds: bool (default - False)
+            whether return timestamps in seconds (default - samples)
+        """
+
+        if not torch.is_tensor(x):
+            try:
+                x = torch.Tensor(x)
+            except:
+                raise TypeError("Audio cannot be casted to tensor. Cast it manually")
+
+        window_size_samples = len(x[0]) if x.dim() == 2 else len(x)
+        self.current_sample += window_size_samples
+
+        speech_prob = self.model(x, self.sampling_rate).item()
+
+        if (speech_prob >= self.threshold) and self.temp_end:
+            self.temp_end = 0
+
+        if (speech_prob >= self.threshold) and not self.triggered:
+            self.triggered = True
+            speech_start = self.current_sample - self.speech_pad_samples
+            return {'start': int(speech_start) if not return_seconds else round(speech_start / self.sampling_rate, 1)}
+
+        if (speech_prob < self.threshold - 0.15) and self.triggered:
+            if not self.temp_end:
+                self.temp_end = self.current_sample
+            if self.current_sample - self.temp_end < self.min_silence_samples:
+                return None
+            else:
+                speech_end = self.temp_end + self.speech_pad_samples
+                self.temp_end = 0
+                self.triggered = False
+                return {'end': int(speech_end) if not return_seconds else round(speech_end / self.sampling_rate, 1)}
+
+        return None
+
+

libs/whisper_streaming/whisper_online.py

@@ -0,0 +1,846 @@
+#!/usr/bin/env python3
+import sys
+import numpy as np
+import librosa
+from functools import lru_cache
+import time
+import logging
+
+import io
+import soundfile as sf
+import math
+
+logger = logging.getLogger(__name__)
+
+@lru_cache
+def load_audio(fname):
+    a, _ = librosa.load(fname, sr=16000, dtype=np.float32)
+    return a
+
+def load_audio_chunk(fname, beg, end):
+    audio = load_audio(fname)
+    beg_s = int(beg*16000)
+    end_s = int(end*16000)
+    return audio[beg_s:end_s]
+
+
+# Whisper backend
+
+class ASRBase:
+
+    sep = " "   # join transcribe words with this character (" " for whisper_timestamped,
+                # "" for faster-whisper because it emits the spaces when neeeded)
+
+    def __init__(self, lan, modelsize=None, cache_dir=None, model_dir=None, logfile=sys.stderr):
+        self.logfile = logfile
+
+        self.transcribe_kargs = {}
+        if lan == "auto":
+            self.original_language = None
+        else:
+            self.original_language = lan
+
+        self.model = self.load_model(modelsize, cache_dir, model_dir)
+
+
+    def load_model(self, modelsize, cache_dir):
+        raise NotImplemented("must be implemented in the child class")
+
+    def transcribe(self, audio, init_prompt=""):
+        raise NotImplemented("must be implemented in the child class")
+
+    def use_vad(self):
+        raise NotImplemented("must be implemented in the child class")
+
+
+class WhisperTimestampedASR(ASRBase):
+    """Uses whisper_timestamped library as the backend. Initially, we tested the code on this backend. It worked, but slower than faster-whisper.
+    On the other hand, the installation for GPU could be easier.
+    """
+
+    sep = " "
+
+    def load_model(self, modelsize=None, cache_dir=None, model_dir=None):
+        import whisper
+        import whisper_timestamped
+        from whisper_timestamped import transcribe_timestamped
+        self.transcribe_timestamped = transcribe_timestamped
+        if model_dir is not None:
+            logger.debug("ignoring model_dir, not implemented")
+        return whisper.load_model(modelsize, download_root=cache_dir)
+
+    def transcribe(self, audio, init_prompt=""):
+        result = self.transcribe_timestamped(self.model,
+                audio, language=self.original_language,
+                initial_prompt=init_prompt, verbose=None,
+                condition_on_previous_text=True, **self.transcribe_kargs)
+        return result
+ 
+    def ts_words(self,r):
+        # return: transcribe result object to [(beg,end,"word1"), ...]
+        o = []
+        for s in r["segments"]:
+            for w in s["words"]:
+                t = (w["start"],w["end"],w["text"])
+                o.append(t)
+        return o
+
+    def segments_end_ts(self, res):
+        return [s["end"] for s in res["segments"]]
+
+    def use_vad(self):
+        self.transcribe_kargs["vad"] = True
+
+    def set_translate_task(self):
+        self.transcribe_kargs["task"] = "translate"
+
+
+
+
+class FasterWhisperASR(ASRBase):
+    """Uses faster-whisper library as the backend. Works much faster, appx 4-times (in offline mode). For GPU, it requires installation with a specific CUDNN version.
+    """
+
+    sep = ""
+
+    def load_model(self, modelsize=None, cache_dir=None, model_dir=None):
+        from faster_whisper import WhisperModel
+#        logging.getLogger("faster_whisper").setLevel(logger.level)
+        if model_dir is not None:
+            logger.debug(f"Loading whisper model from model_dir {model_dir}. modelsize and cache_dir parameters are not used.")
+            model_size_or_path = model_dir
+        elif modelsize is not None:
+            model_size_or_path = modelsize
+        else:
+            raise ValueError("modelsize or model_dir parameter must be set")
+
+
+        # this worked fast and reliably on NVIDIA L40
+        model = WhisperModel(model_size_or_path, device="cuda", compute_type="float16", download_root=cache_dir)
+
+        # or run on GPU with INT8
+        # tested: the transcripts were different, probably worse than with FP16, and it was slightly (appx 20%) slower
+        #model = WhisperModel(model_size, device="cuda", compute_type="int8_float16")
+
+        # or run on CPU with INT8
+        # tested: works, but slow, appx 10-times than cuda FP16
+#        model = WhisperModel(modelsize, device="cpu", compute_type="int8") #, download_root="faster-disk-cache-dir/")
+        return model
+
+    def transcribe(self, audio, init_prompt=""):
+
+        # tested: beam_size=5 is faster and better than 1 (on one 200 second document from En ESIC, min chunk 0.01)
+        segments, info = self.model.transcribe(audio, language=self.original_language, initial_prompt=init_prompt, beam_size=5, word_timestamps=True, condition_on_previous_text=True, **self.transcribe_kargs)
+        #print(info)  # info contains language detection result
+
+        return list(segments)
+
+    def ts_words(self, segments):
+        o = []
+        for segment in segments:
+            for word in segment.words:
+                if segment.no_speech_prob > 0.9:
+                    continue
+                # not stripping the spaces -- should not be merged with them!
+                w = word.word
+                t = (word.start, word.end, w)
+                o.append(t)
+        return o
+
+    def segments_end_ts(self, res):
+        return [s.end for s in res]
+
+    def use_vad(self):
+        self.transcribe_kargs["vad_filter"] = True
+
+    def set_translate_task(self):
+        self.transcribe_kargs["task"] = "translate"
+
+
+class OpenaiApiASR(ASRBase):
+    """Uses OpenAI's Whisper API for audio transcription."""
+
+    def __init__(self, lan=None, temperature=0, logfile=sys.stderr):
+        self.logfile = logfile
+
+        self.modelname = "whisper-1"  
+        self.original_language = None if lan == "auto" else lan # ISO-639-1 language code
+        self.response_format = "verbose_json" 
+        self.temperature = temperature
+
+        self.load_model()
+
+        self.use_vad_opt = False
+
+        # reset the task in set_translate_task
+        self.task = "transcribe"
+
+    def load_model(self, *args, **kwargs):
+        from openai import OpenAI
+        self.client = OpenAI()
+
+        self.transcribed_seconds = 0  # for logging how many seconds were processed by API, to know the cost
+        
+
+    def ts_words(self, segments):
+        no_speech_segments = []
+        if self.use_vad_opt:
+            for segment in segments.segments:
+                # TODO: threshold can be set from outside
+                if segment["no_speech_prob"] > 0.8:
+                    no_speech_segments.append((segment.get("start"), segment.get("end")))
+
+        o = []
+        for word in segments.words:
+            start = word.get("start")
+            end = word.get("end")
+            if any(s[0] <= start <= s[1] for s in no_speech_segments):
+                # print("Skipping word", word.get("word"), "because it's in a no-speech segment")
+                continue
+            o.append((start, end, word.get("word")))
+        return o
+
+
+    def segments_end_ts(self, res):
+        return [s["end"] for s in res.words]
+
+    def transcribe(self, audio_data, prompt=None, *args, **kwargs):
+        # Write the audio data to a buffer
+        buffer = io.BytesIO()
+        buffer.name = "temp.wav"
+        sf.write(buffer, audio_data, samplerate=16000, format='WAV', subtype='PCM_16')
+        buffer.seek(0)  # Reset buffer's position to the beginning
+
+        self.transcribed_seconds += math.ceil(len(audio_data)/16000)  # it rounds up to the whole seconds
+
+        params = {
+            "model": self.modelname,
+            "file": buffer,
+            "response_format": self.response_format,
+            "temperature": self.temperature,
+            "timestamp_granularities": ["word", "segment"]
+        }
+        if self.task != "translate" and self.original_language:
+            params["language"] = self.original_language
+        if prompt:
+            params["prompt"] = prompt
+
+        if self.task == "translate":
+            proc = self.client.audio.translations
+        else:
+            proc = self.client.audio.transcriptions
+
+        # Process transcription/translation
+        transcript = proc.create(**params)
+        logger.debug(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds")
+
+        return transcript
+
+    def use_vad(self):
+        self.use_vad_opt = True
+
+    def set_translate_task(self):
+        self.task = "translate"
+
+
+
+
+class HypothesisBuffer:
+
+    def __init__(self, logfile=sys.stderr):
+        self.commited_in_buffer = []
+        self.buffer = []
+        self.new = []
+
+        self.last_commited_time = 0
+        self.last_commited_word = None
+
+        self.logfile = logfile
+
+    def insert(self, new, offset):
+        # compare self.commited_in_buffer and new. It inserts only the words in new that extend the commited_in_buffer, it means they are roughly behind last_commited_time and new in content
+        # the new tail is added to self.new
+        
+        new = [(a+offset,b+offset,t) for a,b,t in new]
+        self.new = [(a,b,t) for a,b,t in new if a > self.last_commited_time-0.1]
+
+        if len(self.new) >= 1:
+            a,b,t = self.new[0]
+            if abs(a - self.last_commited_time) < 1:
+                if self.commited_in_buffer:
+                    # it's going to search for 1, 2, ..., 5 consecutive words (n-grams) that are identical in commited and new. If they are, they're dropped.
+                    cn = len(self.commited_in_buffer)
+                    nn = len(self.new)
+                    for i in range(1,min(min(cn,nn),5)+1):  # 5 is the maximum 
+                        c = " ".join([self.commited_in_buffer[-j][2] for j in range(1,i+1)][::-1])
+                        tail = " ".join(self.new[j-1][2] for j in range(1,i+1))
+                        if c == tail:
+                            words = []
+                            for j in range(i):
+                                words.append(repr(self.new.pop(0)))
+                            words_msg = " ".join(words)
+                            logger.debug(f"removing last {i} words: {words_msg}")
+                            break
+
+    def flush(self):
+        # returns commited chunk = the longest common prefix of 2 last inserts. 
+
+        commit = []
+        while self.new:
+            na, nb, nt = self.new[0]
+
+            if len(self.buffer) == 0:
+                break
+
+            if nt == self.buffer[0][2]:
+                commit.append((na,nb,nt))
+                self.last_commited_word = nt
+                self.last_commited_time = nb
+                self.buffer.pop(0)
+                self.new.pop(0)
+            else:
+                break
+        self.buffer = self.new
+        self.new = []
+        self.commited_in_buffer.extend(commit)
+        return commit
+
+    def pop_commited(self, time):
+        while self.commited_in_buffer and self.commited_in_buffer[0][1] <= time:
+            self.commited_in_buffer.pop(0)
+
+    def complete(self):
+        return self.buffer
+
+class OnlineASRProcessor:
+
+    SAMPLING_RATE = 16000
+
+    def __init__(self, asr, tokenizer=None, buffer_trimming=("segment", 15), logfile=sys.stderr):
+        """asr: WhisperASR object
+        tokenizer: sentence tokenizer object for the target language. Must have a method *split* that behaves like the one of MosesTokenizer. It can be None, if "segment" buffer trimming option is used, then tokenizer is not used at all.
+        ("segment", 15)
+        buffer_trimming: a pair of (option, seconds), where option is either "sentence" or "segment", and seconds is a number. Buffer is trimmed if it is longer than "seconds" threshold. Default is the most recommended option.
+        logfile: where to store the log. 
+        """
+        self.asr = asr
+        self.tokenizer = tokenizer
+        self.logfile = logfile
+
+        self.init()
+
+        self.buffer_trimming_way, self.buffer_trimming_sec = buffer_trimming
+
+    def init(self, offset=None):
+        """run this when starting or restarting processing"""
+        self.audio_buffer = np.array([],dtype=np.float32)
+        self.transcript_buffer = HypothesisBuffer(logfile=self.logfile)
+        self.buffer_time_offset = 0
+        if offset is not None:
+            self.buffer_time_offset = offset
+        self.transcript_buffer.last_commited_time = self.buffer_time_offset
+        self.commited = []
+
+    def insert_audio_chunk(self, audio):
+        self.audio_buffer = np.append(self.audio_buffer, audio)
+
+    def prompt(self):
+        """Returns a tuple: (prompt, context), where "prompt" is a 200-character suffix of commited text that is inside of the scrolled away part of audio buffer. 
+        "context" is the commited text that is inside the audio buffer. It is transcribed again and skipped. It is returned only for debugging and logging reasons.
+        """
+        k = max(0,len(self.commited)-1)
+        while k > 0 and self.commited[k-1][1] > self.buffer_time_offset:
+            k -= 1
+
+        p = self.commited[:k]
+        p = [t for _,_,t in p]
+        prompt = []
+        l = 0
+        while p and l < 200:  # 200 characters prompt size
+            x = p.pop(-1)
+            l += len(x)+1
+            prompt.append(x)
+        non_prompt = self.commited[k:]
+        return self.asr.sep.join(prompt[::-1]), self.asr.sep.join(t for _,_,t in non_prompt)
+
+    def process_iter(self):
+        """Runs on the current audio buffer.
+        Returns: a tuple (beg_timestamp, end_timestamp, "text"), or (None, None, ""). 
+        The non-emty text is confirmed (committed) partial transcript.
+        """
+
+        prompt, non_prompt = self.prompt()
+        logger.debug(f"PROMPT: {prompt}")
+        logger.debug(f"CONTEXT: {non_prompt}")
+        logger.debug(f"transcribing {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f} seconds from {self.buffer_time_offset:2.2f}")
+        res = self.asr.transcribe(self.audio_buffer, init_prompt=prompt)
+
+        # transform to [(beg,end,"word1"), ...]
+        tsw = self.asr.ts_words(res)
+
+        self.transcript_buffer.insert(tsw, self.buffer_time_offset)
+        o = self.transcript_buffer.flush()
+        self.commited.extend(o)
+        completed = self.to_flush(o)
+        logger.debug(f">>>>COMPLETE NOW: {completed}")
+        the_rest = self.to_flush(self.transcript_buffer.complete())
+        logger.debug(f"INCOMPLETE: {the_rest}")
+
+        # there is a newly confirmed text
+
+        if o and self.buffer_trimming_way == "sentence":  # trim the completed sentences
+            if len(self.audio_buffer)/self.SAMPLING_RATE > self.buffer_trimming_sec:  # longer than this
+                self.chunk_completed_sentence()
+
+        
+        if self.buffer_trimming_way == "segment":
+            s = self.buffer_trimming_sec  # trim the completed segments longer than s,
+        else:
+            s = 30 # if the audio buffer is longer than 30s, trim it
+        
+        if len(self.audio_buffer)/self.SAMPLING_RATE > s:
+            self.chunk_completed_segment(res)
+
+            # alternative: on any word
+            #l = self.buffer_time_offset + len(self.audio_buffer)/self.SAMPLING_RATE - 10
+            # let's find commited word that is less
+            #k = len(self.commited)-1
+            #while k>0 and self.commited[k][1] > l:
+            #    k -= 1
+            #t = self.commited[k][1] 
+            logger.debug("chunking segment")
+            #self.chunk_at(t)
+
+        logger.debug(f"len of buffer now: {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f}")
+        return self.to_flush(o)
+
+    def chunk_completed_sentence(self):
+        if self.commited == []: return
+        logger.debug(self.commited)
+        sents = self.words_to_sentences(self.commited)
+        for s in sents:
+            logger.debug(f"\t\tSENT: {s}")
+        if len(sents) < 2:
+            return
+        while len(sents) > 2:
+            sents.pop(0)
+        # we will continue with audio processing at this timestamp
+        chunk_at = sents[-2][1]
+
+        logger.debug(f"--- sentence chunked at {chunk_at:2.2f}")
+        self.chunk_at(chunk_at)
+
+    def chunk_completed_segment(self, res):
+        if self.commited == []: return
+
+        ends = self.asr.segments_end_ts(res)
+
+        t = self.commited[-1][1]
+
+        if len(ends) > 1:
+
+            e = ends[-2]+self.buffer_time_offset
+            while len(ends) > 2 and e > t:
+                ends.pop(-1)
+                e = ends[-2]+self.buffer_time_offset
+            if e <= t:
+                logger.debug(f"--- segment chunked at {e:2.2f}")
+                self.chunk_at(e)
+            else:
+                logger.debug(f"--- last segment not within commited area")
+        else:
+            logger.debug(f"--- not enough segments to chunk")
+
+
+
+
+
+    def chunk_at(self, time):
+        """trims the hypothesis and audio buffer at "time"
+        """
+        self.transcript_buffer.pop_commited(time)
+        cut_seconds = time - self.buffer_time_offset
+        self.audio_buffer = self.audio_buffer[int(cut_seconds*self.SAMPLING_RATE):]
+        self.buffer_time_offset = time
+
+    def words_to_sentences(self, words):
+        """Uses self.tokenizer for sentence segmentation of words.
+        Returns: [(beg,end,"sentence 1"),...]
+        """
+        
+        cwords = [w for w in words]
+        t = " ".join(o[2] for o in cwords)
+        s = self.tokenizer.split(t)
+        out = []
+        while s:
+            beg = None
+            end = None
+            sent = s.pop(0).strip()
+            fsent = sent
+            while cwords:
+                b,e,w = cwords.pop(0)
+                w = w.strip()
+                if beg is None and sent.startswith(w):
+                    beg = b
+                elif end is None and sent == w:
+                    end = e
+                    out.append((beg,end,fsent))
+                    break
+                sent = sent[len(w):].strip()
+        return out
+
+    def finish(self):
+        """Flush the incomplete text when the whole processing ends.
+        Returns: the same format as self.process_iter()
+        """
+        o = self.transcript_buffer.complete()
+        f = self.to_flush(o)
+        logger.debug(f"last, noncommited: {f}")
+        self.buffer_time_offset += len(self.audio_buffer)/16000
+        return f
+
+
+    def to_flush(self, sents, sep=None, offset=0, ):
+        # concatenates the timestamped words or sentences into one sequence that is flushed in one line
+        # sents: [(beg1, end1, "sentence1"), ...] or [] if empty
+        # return: (beg1,end-of-last-sentence,"concatenation of sentences") or (None, None, "") if empty
+        if sep is None:
+            sep = self.asr.sep
+        t = sep.join(s[2] for s in sents)
+        if len(sents) == 0:
+            b = None
+            e = None
+        else:
+            b = offset + sents[0][0]
+            e = offset + sents[-1][1]
+        return (b,e,t)
+
+class VACOnlineASRProcessor(OnlineASRProcessor):
+    '''Wraps OnlineASRProcessor with VAC (Voice Activity Controller). 
+
+    It works the same way as OnlineASRProcessor: it receives chunks of audio (e.g. 0.04 seconds), 
+    it runs VAD and continuously detects whether there is speech or not. 
+    When it detects end of speech (non-voice for 500ms), it makes OnlineASRProcessor to end the utterance immediately.
+    '''
+
+    def __init__(self, online_chunk_size, *a, **kw):
+        self.online_chunk_size = online_chunk_size
+
+        self.online = OnlineASRProcessor(*a, **kw)
+
+        # VAC:
+        import torch
+        model, _ = torch.hub.load(
+            repo_or_dir='snakers4/silero-vad:v4.0',
+            model='silero_vad'
+        )
+        from silero_vad import VADIterator
+        self.vac = VADIterator(model)  # we use all the default options: 500ms silence, etc.  
+
+        self.logfile = self.online.logfile
+        self.init()
+
+    def init(self):
+        self.online.init()
+        self.vac.reset_states()
+        self.current_online_chunk_buffer_size = 0
+
+        self.is_currently_final = False
+
+        self.status = None  # or "voice" or "nonvoice"
+        self.audio_buffer = np.array([],dtype=np.float32)
+        self.buffer_offset = 0  # in frames
+
+    def clear_buffer(self):
+        self.buffer_offset += len(self.audio_buffer)
+        self.audio_buffer = np.array([],dtype=np.float32)
+
+
+    def insert_audio_chunk(self, audio):
+        res = self.vac(audio)
+        self.audio_buffer = np.append(self.audio_buffer, audio)
+
+        if res is not None:
+            frame = list(res.values())[0]
+            if 'start' in res and 'end' not in res:
+                self.status = 'voice'
+                send_audio = self.audio_buffer[frame-self.buffer_offset:]
+                self.online.init(offset=frame/self.SAMPLING_RATE)
+                self.online.insert_audio_chunk(send_audio)
+                self.current_online_chunk_buffer_size += len(send_audio)
+                self.clear_buffer()
+            elif 'end' in res and 'start' not in res:
+                self.status = 'nonvoice'
+                send_audio = self.audio_buffer[:frame-self.buffer_offset]
+                self.online.insert_audio_chunk(send_audio)
+                self.current_online_chunk_buffer_size += len(send_audio)
+                self.is_currently_final = True
+                self.clear_buffer()
+            else:
+                # It doesn't happen in the current code.
+                raise NotImplemented("both start and end of voice in one chunk!!!")
+        else:
+            if self.status == 'voice':
+                self.online.insert_audio_chunk(self.audio_buffer)
+                self.current_online_chunk_buffer_size += len(self.audio_buffer)
+                self.clear_buffer()
+            else:
+                # We keep 1 second because VAD may later find start of voice in it.
+                # But we trim it to prevent OOM. 
+                self.buffer_offset += max(0,len(self.audio_buffer)-self.SAMPLING_RATE)
+                self.audio_buffer = self.audio_buffer[-self.SAMPLING_RATE:]
+
+
+    def process_iter(self):
+        if self.is_currently_final:
+            return self.finish()
+        elif self.current_online_chunk_buffer_size > self.SAMPLING_RATE*self.online_chunk_size:
+            self.current_online_chunk_buffer_size = 0
+            ret = self.online.process_iter()
+            return ret
+        else:
+            print("no online update, only VAD", self.status, file=self.logfile)
+            return (None, None, "")
+
+    def finish(self):
+        ret = self.online.finish()
+        self.current_online_chunk_buffer_size = 0
+        self.is_currently_final = False
+        return ret
+
+
+
+WHISPER_LANG_CODES = "af,am,ar,as,az,ba,be,bg,bn,bo,br,bs,ca,cs,cy,da,de,el,en,es,et,eu,fa,fi,fo,fr,gl,gu,ha,haw,he,hi,hr,ht,hu,hy,id,is,it,ja,jw,ka,kk,km,kn,ko,la,lb,ln,lo,lt,lv,mg,mi,mk,ml,mn,mr,ms,mt,my,ne,nl,nn,no,oc,pa,pl,ps,pt,ro,ru,sa,sd,si,sk,sl,sn,so,sq,sr,su,sv,sw,ta,te,tg,th,tk,tl,tr,tt,uk,ur,uz,vi,yi,yo,zh".split(",")
+
+def create_tokenizer(lan):
+    """returns an object that has split function that works like the one of MosesTokenizer"""
+
+    assert lan in WHISPER_LANG_CODES, "language must be Whisper's supported lang code: " + " ".join(WHISPER_LANG_CODES)
+
+    if lan == "uk":
+        import tokenize_uk
+        class UkrainianTokenizer:
+            def split(self, text):
+                return tokenize_uk.tokenize_sents(text)
+        return UkrainianTokenizer()
+
+    # supported by fast-mosestokenizer
+    if lan in "as bn ca cs de el en es et fi fr ga gu hi hu is it kn lt lv ml mni mr nl or pa pl pt ro ru sk sl sv ta te yue zh".split():
+        from mosestokenizer import MosesTokenizer
+        return MosesTokenizer(lan)
+
+    # the following languages are in Whisper, but not in wtpsplit:
+    if lan in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split():
+        logger.debug(f"{lan} code is not supported by wtpsplit. Going to use None lang_code option.")
+        lan = None
+
+    from wtpsplit import WtP
+    # downloads the model from huggingface on the first use
+    wtp = WtP("wtp-canine-s-12l-no-adapters")
+    class WtPtok:
+        def split(self, sent):
+            return wtp.split(sent, lang_code=lan)
+    return WtPtok()
+
+
+def add_shared_args(parser):
+    """shared args for simulation (this entry point) and server
+    parser: argparse.ArgumentParser object
+    """
+    parser.add_argument('--min-chunk-size', type=float, default=1.0, help='Minimum audio chunk size in seconds. It waits up to this time to do processing. If the processing takes shorter time, it waits, otherwise it processes the whole segment that was received by this time.')
+    parser.add_argument('--model', type=str, default='large-v2', choices="tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large".split(","),help="Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.")
+    parser.add_argument('--model_cache_dir', type=str, default=None, help="Overriding the default model cache dir where models downloaded from the hub are saved")
+    parser.add_argument('--model_dir', type=str, default=None, help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.")
+    parser.add_argument('--lan', '--language', type=str, default='auto', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.")
+    parser.add_argument('--task', type=str, default='transcribe', choices=["transcribe","translate"],help="Transcribe or translate.")
+    parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped", "openai-api"],help='Load only this backend for Whisper processing.')
+    parser.add_argument('--vac', action="store_true", default=False, help='Use VAC = voice activity controller. Recommended. Requires torch.')
+    parser.add_argument('--vac-chunk-size', type=float, default=0.04, help='VAC sample size in seconds.')
+    parser.add_argument('--vad', action="store_true", default=False, help='Use VAD = voice activity detection, with the default parameters.')
+    parser.add_argument('--buffer_trimming', type=str, default="segment", choices=["sentence", "segment"],help='Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter must be installed for "sentence" option.')
+    parser.add_argument('--buffer_trimming_sec', type=float, default=15, help='Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.')
+    parser.add_argument("-l", "--log-level", dest="log_level", choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help="Set the log level", default='DEBUG')
+
+def asr_factory(args, logfile=sys.stderr):
+    """
+    Creates and configures an ASR and ASR Online instance based on the specified backend and arguments.
+    """
+    backend = args.backend
+    if backend == "openai-api":
+        logger.debug("Using OpenAI API.")
+        asr = OpenaiApiASR(lan=args.lan)
+    else:
+        if backend == "faster-whisper":
+            asr_cls = FasterWhisperASR
+        else:
+            asr_cls = WhisperTimestampedASR
+
+        # Only for FasterWhisperASR and WhisperTimestampedASR
+        size = args.model
+        t = time.time()
+        logger.info(f"Loading Whisper {size} model for {args.lan}...")
+        asr = asr_cls(modelsize=size, lan=args.lan, cache_dir=args.model_cache_dir, model_dir=args.model_dir)
+        e = time.time()
+        logger.info(f"done. It took {round(e-t,2)} seconds.")
+
+    # Apply common configurations
+    if getattr(args, 'vad', False):  # Checks if VAD argument is present and True
+        logger.info("Setting VAD filter")
+        asr.use_vad()
+
+    language = args.lan
+    if args.task == "translate":
+        asr.set_translate_task()
+        tgt_language = "en"  # Whisper translates into English
+    else:
+        tgt_language = language  # Whisper transcribes in this language
+
+    # Create the tokenizer
+    if args.buffer_trimming == "sentence":
+        tokenizer = create_tokenizer(tgt_language)
+    else:
+        tokenizer = None
+
+    # Create the OnlineASRProcessor
+    if args.vac:
+        
+        online = VACOnlineASRProcessor(args.min_chunk_size, asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
+    else:
+        online = OnlineASRProcessor(asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
+
+    return asr, online
+
+def set_logging(args,logger,other="_server"):
+    logging.basicConfig(#format='%(name)s 
+            format='%(levelname)s\t%(message)s')
+    logger.setLevel(args.log_level)
+    logging.getLogger("whisper_online"+other).setLevel(args.log_level)
+#    logging.getLogger("whisper_online_server").setLevel(args.log_level)
+
+
+
+if __name__ == "__main__":
+
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('audio_path', type=str, help="Filename of 16kHz mono channel wav, on which live streaming is simulated.")
+    add_shared_args(parser)
+    parser.add_argument('--start_at', type=float, default=0.0, help='Start processing audio at this time.')
+    parser.add_argument('--offline', action="store_true", default=False, help='Offline mode.')
+    parser.add_argument('--comp_unaware', action="store_true", default=False, help='Computationally unaware simulation.')
+    
+    args = parser.parse_args()
+
+    # reset to store stderr to different file stream, e.g. open(os.devnull,"w")
+    logfile = sys.stderr
+
+    if args.offline and args.comp_unaware:
+        logger.error("No or one option from --offline and --comp_unaware are available, not both. Exiting.")
+        sys.exit(1)
+
+#    if args.log_level:
+#        logging.basicConfig(format='whisper-%(levelname)s:%(name)s: %(message)s',
+#                            level=getattr(logging, args.log_level))
+
+    set_logging(args,logger)
+
+    audio_path = args.audio_path
+
+    SAMPLING_RATE = 16000
+    duration = len(load_audio(audio_path))/SAMPLING_RATE
+    logger.info("Audio duration is: %2.2f seconds" % duration)
+
+    asr, online = asr_factory(args, logfile=logfile)
+    if args.vac:
+        min_chunk = args.vac_chunk_size
+    else:
+        min_chunk = args.min_chunk_size
+
+    # load the audio into the LRU cache before we start the timer
+    a = load_audio_chunk(audio_path,0,1)
+
+    # warm up the ASR because the very first transcribe takes much more time than the other
+    asr.transcribe(a)
+
+    beg = args.start_at
+    start = time.time()-beg
+
+    def output_transcript(o, now=None):
+        # output format in stdout is like:
+        # 4186.3606 0 1720 Takhle to je
+        # - the first three words are:
+        #    - emission time from beginning of processing, in milliseconds
+        #    - beg and end timestamp of the text segment, as estimated by Whisper model. The timestamps are not accurate, but they're useful anyway
+        # - the next words: segment transcript
+        if now is None:
+            now = time.time()-start
+        if o[0] is not None:
+            print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),file=logfile,flush=True)
+            print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),flush=True)
+        else:
+            # No text, so no output
+            pass
+
+    if args.offline: ## offline mode processing (for testing/debugging)
+        a = load_audio(audio_path)
+        online.insert_audio_chunk(a)
+        try:
+            o = online.process_iter()
+        except AssertionError as e:
+            logger.error(f"assertion error: {repr(e)}")
+        else:
+            output_transcript(o)
+        now = None
+    elif args.comp_unaware:  # computational unaware mode 
+        end = beg + min_chunk
+        while True:
+            a = load_audio_chunk(audio_path,beg,end)
+            online.insert_audio_chunk(a)
+            try:
+                o = online.process_iter()
+            except AssertionError as e:
+                logger.error(f"assertion error: {repr(e)}")
+                pass
+            else:
+                output_transcript(o, now=end)
+
+            logger.debug(f"## last processed {end:.2f}s")
+
+            if end >= duration:
+                break
+            
+            beg = end
+            
+            if end + min_chunk > duration:
+                end = duration
+            else:
+                end += min_chunk
+        now = duration
+
+    else: # online = simultaneous mode
+        end = 0
+        while True:
+            now = time.time() - start
+            if now < end+min_chunk:
+                time.sleep(min_chunk+end-now)
+            end = time.time() - start
+            a = load_audio_chunk(audio_path,beg,end)
+            beg = end
+            online.insert_audio_chunk(a)
+
+            try:
+                o = online.process_iter()
+            except AssertionError as e:
+                logger.error(f"assertion error: {e}")
+                pass
+            else:
+                output_transcript(o)
+            now = time.time() - start
+            logger.debug(f"## last processed {end:.2f} s, now is {now:.2f}, the latency is {now-end:.2f}")
+
+            if end >= duration:
+                break
+        now = None
+
+    o = online.finish()
+    output_transcript(o, now=now)

libs/whisper_streaming/whisper_online_server.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+from whisper_online import *
+
+import sys
+import argparse
+import os
+import logging
+import numpy as np
+
+logger = logging.getLogger(__name__)
+parser = argparse.ArgumentParser()
+
+# server options
+parser.add_argument("--host", type=str, default='localhost')
+parser.add_argument("--port", type=int, default=43007)
+parser.add_argument("--warmup-file", type=str, dest="warmup_file", 
+        help="The path to a speech audio wav file to warm up Whisper so that the very first chunk processing is fast. It can be e.g. https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav .")
+
+# options from whisper_online
+add_shared_args(parser)
+args = parser.parse_args()
+
+set_logging(args,logger,other="")
+
+# setting whisper object by args 
+
+SAMPLING_RATE = 16000
+
+size = args.model
+language = args.lan
+asr, online = asr_factory(args)
+min_chunk = args.min_chunk_size
+
+# warm up the ASR because the very first transcribe takes more time than the others. 
+# Test results in https://github.com/ufal/whisper_streaming/pull/81
+msg = "Whisper is not warmed up. The first chunk processing may take longer."
+if args.warmup_file:
+    if os.path.isfile(args.warmup_file):
+        a = load_audio_chunk(args.warmup_file,0,1)
+        asr.transcribe(a)
+        logger.info("Whisper is warmed up.")
+    else:
+        logger.critical("The warm up file is not available. "+msg)
+        sys.exit(1)
+else:
+    logger.warning(msg)
+
+
+######### Server objects
+
+import line_packet
+import socket
+
+class Connection:
+    '''it wraps conn object'''
+    PACKET_SIZE = 32000*5*60 # 5 minutes # was: 65536
+
+    def __init__(self, conn):
+        self.conn = conn
+        self.last_line = ""
+
+        self.conn.setblocking(True)
+
+    def send(self, line):
+        '''it doesn't send the same line twice, because it was problematic in online-text-flow-events'''
+        if line == self.last_line:
+            return
+        line_packet.send_one_line(self.conn, line)
+        self.last_line = line
+
+    def receive_lines(self):
+        in_line = line_packet.receive_lines(self.conn)
+        return in_line
+
+    def non_blocking_receive_audio(self):
+        try:
+            r = self.conn.recv(self.PACKET_SIZE)
+            return r
+        except ConnectionResetError:
+            return None
+
+
+import io
+import soundfile
+
+# wraps socket and ASR object, and serves one client connection. 
+# next client should be served by a new instance of this object
+class ServerProcessor:
+
+    def __init__(self, c, online_asr_proc, min_chunk):
+        self.connection = c
+        self.online_asr_proc = online_asr_proc
+        self.min_chunk = min_chunk
+
+        self.last_end = None
+
+        self.is_first = True
+
+    def receive_audio_chunk(self):
+        # receive all audio that is available by this time
+        # blocks operation if less than self.min_chunk seconds is available
+        # unblocks if connection is closed or a chunk is available
+        out = []
+        minlimit = self.min_chunk*SAMPLING_RATE
+        while sum(len(x) for x in out) < minlimit:
+            raw_bytes = self.connection.non_blocking_receive_audio()
+            if not raw_bytes:
+                break
+#            print("received audio:",len(raw_bytes), "bytes", raw_bytes[:10])
+            sf = soundfile.SoundFile(io.BytesIO(raw_bytes), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW")
+            audio, _ = librosa.load(sf,sr=SAMPLING_RATE,dtype=np.float32)
+            out.append(audio)
+        if not out:
+            return None
+        conc = np.concatenate(out)
+        if self.is_first and len(conc) < minlimit:
+            return None
+        self.is_first = False
+        return np.concatenate(out)
+
+    def format_output_transcript(self,o):
+        # output format in stdout is like:
+        # 0 1720 Takhle to je
+        # - the first two words are:
+        #    - beg and end timestamp of the text segment, as estimated by Whisper model. The timestamps are not accurate, but they're useful anyway
+        # - the next words: segment transcript
+
+        # This function differs from whisper_online.output_transcript in the following:
+        # succeeding [beg,end] intervals are not overlapping because ELITR protocol (implemented in online-text-flow events) requires it.
+        # Therefore, beg, is max of previous end and current beg outputed by Whisper.
+        # Usually it differs negligibly, by appx 20 ms.
+
+        if o[0] is not None:
+            beg, end = o[0]*1000,o[1]*1000
+            if self.last_end is not None:
+                beg = max(beg, self.last_end)
+
+            self.last_end = end
+            print("%1.0f %1.0f %s" % (beg,end,o[2]),flush=True,file=sys.stderr)
+            return "%1.0f %1.0f %s" % (beg,end,o[2])
+        else:
+            logger.debug("No text in this segment")
+            return None
+
+    def send_result(self, o):
+        msg = self.format_output_transcript(o)
+        if msg is not None:
+            self.connection.send(msg)
+
+    def process(self):
+        # handle one client connection
+        self.online_asr_proc.init()
+        while True:
+            a = self.receive_audio_chunk()
+            if a is None:
+                break
+            self.online_asr_proc.insert_audio_chunk(a)
+            o = online.process_iter()
+            try:
+                self.send_result(o)
+            except BrokenPipeError:
+                logger.info("broken pipe -- connection closed?")
+                break
+
+#        o = online.finish()  # this should be working
+#        self.send_result(o)
+
+
+
+# server loop
+
+with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+    s.bind((args.host, args.port))
+    s.listen(1)
+    logger.info('Listening on'+str((args.host, args.port)))
+    while True:
+        conn, addr = s.accept()
+        logger.info('Connected to client on {}'.format(addr))
+        connection = Connection(conn)
+        proc = ServerProcessor(connection, online, args.min_chunk_size)
+        proc.process()
+        conn.close()
+        logger.info('Connection to client closed')
+logger.info('Connection closed, terminating.')