updated

README.md

@@ -0,0 +1,66 @@
+---
+title: Whisper Webui
+emoji: ⚡
+colorFrom: pink
+colorTo: purple
+sdk: gradio
+sdk_version: 3.3.1
+app_file: app.py
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# Running Locally
+
+To run this program locally, first install Python 3.9+ and Git. Then install Pytorch 10.1+ and all the other dependencies:
+```
+pip install -r requirements.txt
+```
+
+Finally, run the full version (no audio length restrictions) of the app:
+```
+python app-full.py
+```
+
+You can also run the CLI interface, which is similar to Whisper's own CLI but also supports the following additional arguments:
+```
+python cli.py \
+[--vad {none,silero-vad,silero-vad-skip-gaps,silero-vad-expand-into-gaps,periodic-vad}] \
+[--vad_merge_window VAD_MERGE_WINDOW] \
+[--vad_max_merge_size VAD_MAX_MERGE_SIZE] \
+[--vad_padding VAD_PADDING] \
+[--vad_prompt_window VAD_PROMPT_WINDOW]
+```
+In addition, you may also use URL's in addition to file paths as input.
+```
+python cli.py --model large --vad silero-vad --language Japanese "https://www.youtube.com/watch?v=4cICErqqRSM"
+```
+
+# Docker
+
+To run it in Docker, first install Docker and optionally the NVIDIA Container Toolkit in order to use the GPU. Then 
+check out this repository and build an image:
+```
+sudo docker build -t whisper-webui:1 .
+```
+
+You can then start the WebUI with GPU support like so:
+```
+sudo docker run -d --gpus=all -p 7860:7860 whisper-webui:1
+```
+
+Leave out "--gpus=all" if you don't have access to a GPU with enough memory, and are fine with running it on the CPU only:
+```
+sudo docker run -d -p 7860:7860 whisper-webui:1
+```
+
+## Caching
+
+Note that the models themselves are currently not included in the Docker images, and will be downloaded on the demand.
+To avoid this, bind the directory /root/.cache/whisper to some directory on the host (for instance /home/administrator/.cache/whisper), where you can (optionally) 
+prepopulate the directory with the different Whisper models. 
+```
+sudo docker run -d --gpus=all -p 7860:7860 --mount type=bind,source=/home/administrator/.cache/whisper,target=/root/.cache/whisper whisper-webui:1
+```

app-local.py

@@ -0,0 +1,3 @@
+# Run the app with no audio file restrictions
+from app import create_ui
+create_ui(-1)

app-network.py

@@ -0,0 +1,3 @@
+# Run the app with no audio file restrictions, and make it available on the network
+from app import create_ui
+create_ui(-1, server_name="0.0.0.0")

app-shared.py

@@ -0,0 +1,3 @@
+# Run the app with no audio file restrictions
+from app import create_ui
+create_ui(-1, share=True)

app.py

@@ -0,0 +1,263 @@
+from typing import Iterator
+
+from io import StringIO
+import os
+import pathlib
+import tempfile
+
+# External programs
+import whisper
+import ffmpeg
+
+# UI
+import gradio as gr
+
+from src.download import ExceededMaximumDuration, download_url
+from src.utils import slugify, write_srt, write_vtt
+from src.vad import NonSpeechStrategy, PeriodicTranscriptionConfig, TranscriptionConfig, VadPeriodicTranscription, \
+    VadSileroTranscription
+
+# Limitations (set to -1 to disable)
+DEFAULT_INPUT_AUDIO_MAX_DURATION = 600  # seconds
+
+# Whether or not to automatically delete all uploaded files, to save disk space
+DELETE_UPLOADED_FILES = True
+
+# Gradio seems to truncate files without keeping the extension, so we need to truncate the file prefix ourself 
+MAX_FILE_PREFIX_LENGTH = 17
+
+LANGUAGES = [
+    "English", "Hindi", "Tamil", "Urdu",
+    "Malayalam", "Telugu", "Bengali", "Kannada",
+    "Nepali", "Marathi", "Punjabi", "Sindhi",
+    "Gujarati", "Sanskrit", "Assamese"]
+
+
+class WhisperTranscriber:
+    def __init__(self, inputAudioMaxDuration: float = DEFAULT_INPUT_AUDIO_MAX_DURATION,
+                 deleteUploadedFiles: bool = DELETE_UPLOADED_FILES):
+        self.model_cache = dict()
+
+        self.vad_model = None
+        self.inputAudioMaxDuration = inputAudioMaxDuration
+        self.deleteUploadedFiles = deleteUploadedFiles
+
+    def transcribe_webui(self, modelName, languageName, urlData, uploadFile, microphoneData, task, vad, vadMergeWindow,
+                         vadMaxMergeSize, vadPadding, vadPromptWindow):
+        try:
+            source, sourceName = self.__get_source(urlData, uploadFile, microphoneData)
+
+            try:
+                selectedLanguage = languageName.lower() if len(languageName) > 0 else None
+                selectedModel = modelName if modelName is not None else "base"
+
+                model = self.model_cache.get(selectedModel, None)
+
+                if not model:
+                    model = whisper.load_model(selectedModel)
+                    self.model_cache[selectedModel] = model
+
+                # Execute whisper
+                result = self.transcribe_file(model, source, selectedLanguage, task, vad, vadMergeWindow,
+                                              vadMaxMergeSize, vadPadding, vadPromptWindow)
+
+                # Write result
+                downloadDirectory = tempfile.mkdtemp()
+
+                filePrefix = slugify(sourceName, allow_unicode=True)
+                download, text, vtt = self.write_result(result, filePrefix, downloadDirectory)
+
+                return download, text, vtt
+
+            finally:
+                # Cleanup source
+                if self.deleteUploadedFiles:
+                    print("Deleting source file " + source)
+                    os.remove(source)
+
+        except ExceededMaximumDuration as e:
+            return [], ("[ERROR]: Maximum remote video length is " + str(e.maxDuration) + "s, file was " + str(
+                e.videoDuration) + "s"), "[ERROR]"
+
+    def transcribe_file(self, model: whisper.Whisper, audio_path: str, language: str, task: str = None, vad: str = None,
+                        vadMergeWindow: float = 5, vadMaxMergeSize: float = 150, vadPadding: float = 1,
+                        vadPromptWindow: float = 1, **decodeOptions: dict):
+
+        initial_prompt = decodeOptions.pop('initial_prompt', None)
+
+        if ('task' in decodeOptions):
+            task = decodeOptions.pop('task')
+
+        # Callable for processing an audio file
+        whisperCallable = lambda audio, segment_index, prompt, detected_language: model.transcribe(audio, \
+                                                                                                   language=language if language else detected_language,
+                                                                                                   task=task, \
+                                                                                                   initial_prompt=self._concat_prompt(
+                                                                                                       initial_prompt,
+                                                                                                       prompt) if segment_index == 0 else prompt, \
+                                                                                                   **decodeOptions)
+
+        # The results
+        if (vad == 'silero-vad'):
+            # Silero VAD where non-speech gaps are transcribed
+            process_gaps = self._create_silero_config(NonSpeechStrategy.CREATE_SEGMENT, vadMergeWindow, vadMaxMergeSize,
+                                                      vadPadding, vadPromptWindow)
+            result = self.vad_model.transcribe(audio_path, whisperCallable, process_gaps)
+        elif (vad == 'silero-vad-skip-gaps'):
+            # Silero VAD where non-speech gaps are simply ignored
+            skip_gaps = self._create_silero_config(NonSpeechStrategy.SKIP, vadMergeWindow, vadMaxMergeSize, vadPadding,
+                                                   vadPromptWindow)
+            result = self.vad_model.transcribe(audio_path, whisperCallable, skip_gaps)
+        elif (vad == 'silero-vad-expand-into-gaps'):
+            # Use Silero VAD where speech-segments are expanded into non-speech gaps
+            expand_gaps = self._create_silero_config(NonSpeechStrategy.EXPAND_SEGMENT, vadMergeWindow, vadMaxMergeSize,
+                                                     vadPadding, vadPromptWindow)
+            result = self.vad_model.transcribe(audio_path, whisperCallable, expand_gaps)
+        elif (vad == 'periodic-vad'):
+            # Very simple VAD - mark every 5 minutes as speech. This makes it less likely that Whisper enters an infinite loop, but
+            # it may create a break in the middle of a sentence, causing some artifacts.
+            periodic_vad = VadPeriodicTranscription()
+            result = periodic_vad.transcribe(audio_path, whisperCallable,
+                                             PeriodicTranscriptionConfig(periodic_duration=vadMaxMergeSize,
+                                                                         max_prompt_window=vadPromptWindow))
+        else:
+            # Default VAD
+            result = whisperCallable(audio_path, 0, None, None)
+
+        return result
+
+    def _concat_prompt(self, prompt1, prompt2):
+        if (prompt1 is None):
+            return prompt2
+        elif (prompt2 is None):
+            return prompt1
+        else:
+            return prompt1 + " " + prompt2
+
+    def _create_silero_config(self, non_speech_strategy: NonSpeechStrategy, vadMergeWindow: float = 5,
+                              vadMaxMergeSize: float = 150, vadPadding: float = 1, vadPromptWindow: float = 1):
+        # Use Silero VAD 
+        if (self.vad_model is None):
+            self.vad_model = VadSileroTranscription()
+
+        config = TranscriptionConfig(non_speech_strategy=non_speech_strategy,
+                                     max_silent_period=vadMergeWindow, max_merge_size=vadMaxMergeSize,
+                                     segment_padding_left=vadPadding, segment_padding_right=vadPadding,
+                                     max_prompt_window=vadPromptWindow)
+
+        return config
+
+    def write_result(self, result: dict, source_name: str, output_dir: str):
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        text = result["text"]
+        language = result["language"]
+        languageMaxLineWidth = self.__get_max_line_width(language)
+
+        print("Max line width " + str(languageMaxLineWidth))
+        vtt = self.__get_subs(result["segments"], "vtt", languageMaxLineWidth)
+        srt = self.__get_subs(result["segments"], "srt", languageMaxLineWidth)
+
+        output_files = []
+        output_files.append(self.__create_file(srt, output_dir, source_name + "-subs.srt"));
+        output_files.append(self.__create_file(vtt, output_dir, source_name + "-subs.vtt"));
+        output_files.append(self.__create_file(text, output_dir, source_name + "-transcript.txt"));
+
+        return output_files, text, vtt
+
+    def clear_cache(self):
+        self.model_cache = dict()
+        self.vad_model = None
+
+    def __get_source(self, urlData, uploadFile, microphoneData):
+        if urlData:
+            # Download from YouTube
+            source = download_url(urlData, self.inputAudioMaxDuration)[0]
+        else:
+            # File input
+            source = uploadFile if uploadFile is not None else microphoneData
+
+            if self.inputAudioMaxDuration > 0:
+                # Calculate audio length
+                audioDuration = ffmpeg.probe(source)["format"]["duration"]
+
+                if float(audioDuration) > self.inputAudioMaxDuration:
+                    raise ExceededMaximumDuration(videoDuration=audioDuration, maxDuration=self.inputAudioMaxDuration,
+                                                  message="Video is too long")
+
+        file_path = pathlib.Path(source)
+        sourceName = file_path.stem[:MAX_FILE_PREFIX_LENGTH] + file_path.suffix
+
+        return source, sourceName
+
+    def __get_max_line_width(self, language: str) -> int:
+        if (language and language.lower() in ["japanese", "ja", "chinese", "zh"]):
+            # Chinese characters and kana are wider, so limit line length to 40 characters
+            return 40
+        else:
+            # TODO: Add more languages
+            # 80 latin characters should fit on a 1080p/720p screen
+            return 80
+
+    def __get_subs(self, segments: Iterator[dict], format: str, maxLineWidth: int) -> str:
+        segmentStream = StringIO()
+
+        if format == 'vtt':
+            write_vtt(segments, file=segmentStream, maxLineWidth=maxLineWidth)
+        elif format == 'srt':
+            write_srt(segments, file=segmentStream, maxLineWidth=maxLineWidth)
+        else:
+            raise Exception("Unknown format " + format)
+
+        segmentStream.seek(0)
+        return segmentStream.read()
+
+    def __create_file(self, text: str, directory: str, fileName: str) -> str:
+        # Write the text to a file
+        with open(os.path.join(directory, fileName), 'w+', encoding="utf-8") as file:
+            file.write(text)
+
+        return file.name
+
+
+def create_ui(inputAudioMaxDuration, share=False, server_name: str = None):
+    ui = WhisperTranscriber(inputAudioMaxDuration)
+
+    ui_description = "NS AI LABS CUSTOMISED WHISPER WITH CUSTOM ASR LAYERS "
+    ui_description += "YOU SPEAK IN ANY INDIAN LANGUAGE AT ANY CONDITION & PACE , WE WILL GIVE YOU BEST CONTEXTUAL " \
+                      "ASR IN ENGLISH "
+    ui_description += " SUPPORTS HIN-ENGLISH TOO"
+
+    ui_description += "\n\n\n\nFor longer audio files (>10 minutes) not in English, it is recommended that you select Silero VAD (Voice Activity Detector) in the VAD option."
+
+    if inputAudioMaxDuration > 0:
+        ui_description += "\n\n" + "Max audio file length: " + str(inputAudioMaxDuration) + " s"
+
+    ui_article = "CHOOSE SMALL MODEL ( WE CUSTOMISED THIS ONLY )"
+
+    demo = gr.Interface(fn=ui.transcribe_webui, description=ui_description, article=ui_article, inputs=[
+        gr.Dropdown(choices=["tiny", "base", "small", "medium", "large"], value="medium", label="Model"),
+        gr.Dropdown(choices=sorted(LANGUAGES), label="Language"),
+        gr.Text(label="URL (cloudfont URL, etc.)"),
+        gr.Audio(source="upload", type="filepath", label="Upload Audio"),
+        gr.Audio(source="microphone", type="filepath", label="Microphone Input"),
+        gr.Dropdown(choices=["transcribe", "translate"], label="Task"),
+        gr.Dropdown(
+            choices=["none", "silero-vad", "silero-vad-skip-gaps", "silero-vad-expand-into-gaps", "periodic-vad"],
+            label="VAD"),
+        gr.Number(label="VAD - Merge Window (s)", precision=0, value=5),
+        gr.Number(label="VAD - Max Merge Size (s)", precision=0, value=30),
+        gr.Number(label="VAD - Padding (s)", precision=None, value=1),
+        gr.Number(label="VAD - Prompt Window (s)", precision=None, value=3)
+    ], outputs=[
+        gr.File(label="Download"),
+        gr.Text(label="Transcription"),
+        gr.Text(label="Segments")
+    ])
+
+    demo.launch(share=share, server_name=server_name)
+
+
+if __name__ == '__main__':
+    create_ui(DEFAULT_INPUT_AUDIO_MAX_DURATION)

cli.py

@@ -0,0 +1,109 @@
+import argparse
+import os
+from urllib.parse import urlparse
+import warnings
+import numpy as np
+
+import whisper
+
+import torch
+from app import LANGUAGES, WhisperTranscriber
+from src.download import download_url
+
+from src.utils import optional_float, optional_int, str2bool
+
+
+def cli():
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("audio", nargs="+", type=str, help="audio file(s) to transcribe")
+    parser.add_argument("--model", default="small", choices=["tiny", "base", "small", "medium", "large"], help="name of the Whisper model to use")
+    parser.add_argument("--model_dir", type=str, default=None, help="the path to save model files; uses ~/.cache/whisper by default")
+    parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu", help="device to use for PyTorch inference")
+    parser.add_argument("--output_dir", "-o", type=str, default=".", help="directory to save the outputs")
+    parser.add_argument("--verbose", type=str2bool, default=True, help="whether to print out the progress and debug messages")
+
+    parser.add_argument("--task", type=str, default="transcribe", choices=["transcribe", "translate"], help="whether to perform X->X speech recognition ('transcribe') or X->English translation ('translate')")
+    parser.add_argument("--language", type=str, default=None, choices=sorted(LANGUAGES), help="language spoken in the audio, specify None to perform language detection")
+
+    parser.add_argument("--vad", type=str, default="none", choices=["none", "silero-vad", "silero-vad-skip-gaps", "silero-vad-expand-into-gaps", "periodic-vad"], help="The voice activity detection algorithm to use")
+    parser.add_argument("--vad_merge_window", type=optional_float, default=5, help="The window size (in seconds) to merge voice segments")
+    parser.add_argument("--vad_max_merge_size", type=optional_float, default=30, help="The maximum size (in seconds) of a voice segment")
+    parser.add_argument("--vad_padding", type=optional_float, default=1, help="The padding (in seconds) to add to each voice segment")
+    parser.add_argument("--vad_prompt_window", type=optional_float, default=3, help="The window size of the prompt to pass to Whisper")
+
+    parser.add_argument("--temperature", type=float, default=0, help="temperature to use for sampling")
+    parser.add_argument("--best_of", type=optional_int, default=5, help="number of candidates when sampling with non-zero temperature")
+    parser.add_argument("--beam_size", type=optional_int, default=5, help="number of beams in beam search, only applicable when temperature is zero")
+    parser.add_argument("--patience", type=float, default=None, help="optional patience value to use in beam decoding, as in https://arxiv.org/abs/2204.05424, the default (1.0) is equivalent to conventional beam search")
+    parser.add_argument("--length_penalty", type=float, default=None, help="optional token length penalty coefficient (alpha) as in https://arxiv.org/abs/1609.08144, uses simple lengt normalization by default")
+
+    parser.add_argument("--suppress_tokens", type=str, default="-1", help="comma-separated list of token ids to suppress during sampling; '-1' will suppress most special characters except common punctuations")
+    parser.add_argument("--initial_prompt", type=str, default=None, help="optional text to provide as a prompt for the first window.")
+    parser.add_argument("--condition_on_previous_text", type=str2bool, default=True, help="if True, provide the previous output of the model as a prompt for the next window; disabling may make the text inconsistent across windows, but the model becomes less prone to getting stuck in a failure loop")
+    parser.add_argument("--fp16", type=str2bool, default=True, help="whether to perform inference in fp16; True by default")
+
+    parser.add_argument("--temperature_increment_on_fallback", type=optional_float, default=0.2, help="temperature to increase when falling back when the decoding fails to meet either of the thresholds below")
+    parser.add_argument("--compression_ratio_threshold", type=optional_float, default=2.4, help="if the gzip compression ratio is higher than this value, treat the decoding as failed")
+    parser.add_argument("--logprob_threshold", type=optional_float, default=-1.0, help="if the average log probability is lower than this value, treat the decoding as failed")
+    parser.add_argument("--no_speech_threshold", type=optional_float, default=0.6, help="if the probability of the <|nospeech|> token is higher than this value AND the decoding has failed due to `logprob_threshold`, consider the segment as silence")
+
+    args = parser.parse_args().__dict__
+    model_name: str = args.pop("model")
+    model_dir: str = args.pop("model_dir")
+    output_dir: str = args.pop("output_dir")
+    device: str = args.pop("device")
+    os.makedirs(output_dir, exist_ok=True)
+
+    if model_name.endswith(".en") and args["language"] not in {"en", "English"}:
+        warnings.warn(f"{model_name} is an English-only model but receipted '{args['language']}'; using English instead.")
+        args["language"] = "en"
+
+    temperature = args.pop("temperature")
+    temperature_increment_on_fallback = args.pop("temperature_increment_on_fallback")
+    if temperature_increment_on_fallback is not None:
+        temperature = tuple(np.arange(temperature, 1.0 + 1e-6, temperature_increment_on_fallback))
+    else:
+        temperature = [temperature]
+
+    vad = args.pop("vad")
+    vad_merge_window = args.pop("vad_merge_window")
+    vad_max_merge_size = args.pop("vad_max_merge_size")
+    vad_padding = args.pop("vad_padding")
+    vad_prompt_window = args.pop("vad_prompt_window")
+
+    model = whisper.load_model(model_name, device=device, download_root=model_dir)
+    transcriber = WhisperTranscriber(deleteUploadedFiles=False)
+
+    for audio_path in args.pop("audio"):
+        sources = []
+
+        # Detect URL and download the audio
+        if (uri_validator(audio_path)):
+            # Download from YouTube/URL directly
+            for source_path in  download_url(audio_path, maxDuration=-1, destinationDirectory=output_dir, playlistItems=None):
+                source_name = os.path.basename(source_path)
+                sources.append({ "path": source_path, "name": source_name })
+        else:
+            sources.append({ "path": audio_path, "name": os.path.basename(audio_path) })
+
+        for source in sources:
+            source_path = source["path"]
+            source_name = source["name"]
+
+            result = transcriber.transcribe_file(model, source_path, temperature=temperature, 
+                                                vad=vad, vadMergeWindow=vad_merge_window, vadMaxMergeSize=vad_max_merge_size, 
+                                                vadPadding=vad_padding, vadPromptWindow=vad_prompt_window, **args)
+            
+            transcriber.write_result(result, source_name, output_dir)
+
+    transcriber.clear_cache()
+
+def uri_validator(x):
+    try:
+        result = urlparse(x)
+        return all([result.scheme, result.netloc])
+    except:
+        return False
+
+if __name__ == '__main__':
+    cli()

dockerfile

@@ -0,0 +1,20 @@
+FROM huggingface/transformers-pytorch-gpu
+EXPOSE 7860
+
+ADD . /opt/whisper-webui/
+
+# Latest version of transformers-pytorch-gpu seems to lack tk. 
+# Further, pip install fails, so we must upgrade pip first.
+RUN apt-get -y install python3-tk
+RUN  python3 -m pip install --upgrade pip &&\
+     python3 -m pip install -r /opt/whisper-webui/requirements.txt
+
+# Note: Models will be downloaded on demand to the directory /root/.cache/whisper.
+# You can also bind this directory in the container to somewhere on the host.
+
+# To be able to see logs in real time
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /opt/whisper-webui/
+ENTRYPOINT ["python3"]
+CMD ["app-network.py"]

requirements.txt

@@ -0,0 +1,6 @@
+git+https://github.com/openai/whisper.git
+transformers
+ffmpeg-python==0.2.0
+gradio
+yt-dlp
+torchaudio