Update app.py

app.py

@@ -18,27 +18,110 @@ Description: webapp, transkribering (norsk), NbAiLab/nb-whisper-large, oppsummer
 
 import time
 import os
+import re
 import warnings
 from pydub import AudioSegment
+
+import pandas as pd
+import numpy as np
 import torch
 import torchaudio
 import torchaudio.transforms as transforms
+
+
 from transformers import AutoProcessor, AutoModelForSpeechSeq2Seq
-from huggingface_hub import model_info
+from ...generation.configuration_utils import GenerationConfig
+
 import spacy
 import networkx as nx
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
-import pandas as pd
-import numpy as np
-import re
+
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+
 import gradio as gr
 from fpdf import FPDF
 from PIL import Image
+# from huggingface_hub import model_info
 
+#############################################################################################################################################3
 # Suppress warnings
 warnings.filterwarnings("ignore")
+"""
+def generate(
+        self,
+        input_features: Optional[torch.Tensor] = None, # <====================== ACTIVE
+        generation_config: Optional[GenerationConfig] = None, # <====================== could be ACTIVE(ed.)*
+        logits_processor: Optional[LogitsProcessorList] = None,
+        stopping_criteria: Optional[StoppingCriteriaList] = None,
+        prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None,
+        synced_gpus: bool = False,
+        return_timestamps: Optional[bool] = None,
+        task: Optional[str] = None,
+        language: Optional[Union[str, List[str]]] = None, # <====================== ACTIVE
+        is_multilingual: Optional[bool] = None,
+        prompt_ids: Optional[torch.Tensor] = None,
+        prompt_condition_type: Optional[str] = None,  # first-segment, all-segments
+        condition_on_prev_tokens: Optional[bool] = None,
+        temperature: Optional[Union[float, Tuple[float, ...]]] = None,
+        compression_ratio_threshold: Optional[float] = None,
+        logprob_threshold: Optional[float] = None,
+        no_speech_threshold: Optional[float] = None,
+        num_segment_frames: Optional[int] = None,
+        attention_mask: Optional[torch.Tensor] = None,  # <====================== NOT ACTIVE by DEFAULT
+        time_precision: float = 0.02,
+        return_token_timestamps: Optional[bool] = None,
+        return_segments: bool = False,
+        return_dict_in_generate: Optional[bool] = None,
+        **kwargs, # <====================== ACTIVE
+    ):
+"""
+        """
+        *generation_config (`~generation.GenerationConfig`, *optional*):
+                The generation configuration to be used as base parametrization for the generation call. `**kwargs`
+                passed to generate matching the attributes of `generation_config` will override them. If
+                `generation_config` is not provided, the default will be used, which had the following loading
+                priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
+                configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
+                default values, whose documentation should be checked to parameterize generation.
+
+        from v4.39 the forced decoder ids are always None in favour of decoder input ids
+        generation_config.forced_decoder_ids = None
+        """
+
+"""
+Example:
+
+        - *Longform transcription*: To transcribe or translate audios longer than 30 seconds, process the audio files without truncation and pass all mel features at once to generate.
+
+        ```python
+        >>> import torch
+        >>> from transformers import AutoProcessor, WhisperForConditionalGeneration
+        >>> from datasets import load_dataset, Audio
+
+        >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en")
+        >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en")
+        >>> model.cuda()  # doctest: +IGNORE_RESULT
+
+        >>> # load audios > 30 seconds
+        >>> ds = load_dataset("distil-whisper/meanwhile", "default")["test"]
+        >>> # resample to 16kHz
+        >>> ds = ds.cast_column("audio", Audio(sampling_rate=16000))
+        >>> # take first 8 audios and retrieve array
+        >>> audio = ds[:8]["audio"]
+        >>> audio = [x["array"] for x in audio]
+
+        >>> # make sure to NOT truncate the input audio, to return the `attention_mask` and to pad to the longest audio
+        >>> inputs = processor(audio, return_tensors="pt", truncation=False, padding="longest", return_attention_mask=True, sampling_rate=16_000)
+        >>> inputs = inputs.to("cuda", torch.float32)
+
+        >>> # transcribe audio to ids
+        >>> generated_ids = model.generate(**inputs)
+
+        >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)
+        >>> transcription[0]
+        " Folks, if you watch the show, you know, I spent a lot of time (..)"
+"""
 
 # Convert m4a audio to wav format
 def convert_to_wav(audio_file):
@@ -46,43 +129,61 @@ def convert_to_wav(audio_file):
     wav_file = "temp.wav"
     audio.export(wav_file, format="wav")
     return wav_file
-
-# Initialize device for torch
-device = 0 if torch.cuda.is_available() else "cpu"
-torch_dtype = torch.float32
-
-# Load tokenizer and model
+#############################################################################################################################################3
+#
+#
+#
+#
+#---------------------------------------------------------------------------------------------------------------------------------------------
 processor = AutoProcessor.from_pretrained("NbAiLab/nb-whisper-large-verbatim")
 model = AutoModelForSpeechSeq2Seq.from_pretrained("NbAiLab/nb-whisper-large-verbatim")
+model.cuda() # device = 0 if torch.cuda.is_available() else "cpu"
 
 
-# Model script does not support JIT compilation
-#model = model.to(device)
-#model = torch.jit.script(model)
+ # 0. deprecate old inputs
+    if "inputs" in kwargs:
+        input_features = kwargs.pop("inputs")
+        warnings.warn(
+            "The input name `inputs` is deprecated. Please make sure to use `input_features` instead.",
+            FutureWarning,
+        )
+"""
+    # 1. prepare generation config
+    generation_config, kwargs = self._prepare_generation_config(generation_config, **kwargs)
+
+    # 2. set global generate variables
+    #input_stride = self.model.encoder.conv1.stride[0] * self.model.encoder.conv2.stride[0]
+    #num_segment_frames = input_stride * self.config.max_source_positions
+    #batch_size, total_input_frames = self._retrieve_total_input_frames(
+        input_features=input_features, kwargs=kwargs #input_stride=input_stride,
+    )
+"""
 
-# Generation kwargs
 generate_kwargs = {
     "num_beams": 5,
-    "task": "transcribe",
     "language": "no",
-    "forced_decoder_ids": None
+    "task": "transcribe",
+    "forced_decoder_ids": None # ALT. generation_config.forced_decoder_ids = None
 }
 
-# Transcribe
 def transcribe_audio(audio_file, chunk_length_s=30):
+#---------------------------------------------------------------------------------------------------------------------------------------------
+#
+#
+#
+#
+#############################################################################################################################################3
     if audio_file.endswith(".m4a"):
         audio_file = convert_to_wav(audio_file)
 
     start_time = time.time()
-
-    # Load the audio waveform using torchaudio
+    # Load waveform using torchaudio
     waveform, sample_rate = torchaudio.load(audio_file)
 
     # Convert to mono if the audio has more than one channel
     if waveform.shape[0] > 1:
         waveform = torch.mean(waveform, dim=0, keepdim=True)
 
-    # Resample audio to 16000 Hz if it’s not already
     if sample_rate != 16000:
         resampler = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=16000)
         waveform = resampler(waveform)
@@ -103,28 +204,30 @@ def transcribe_audio(audio_file, chunk_length_s=30):
         # Check chunk waveform is properly shaped
         if chunk_waveform.shape[0] > 1:
             chunk_waveform = torch.mean(chunk_waveform, dim=0, keepdim=True)
+#############################################################################################################################################3
+#
+#
+#
+#
+#---------------------------------------------------------------------------------------------------------------------------------------------
 
-        if processor.tokenizer.pad_token is None or processor.tokenizer.pad_token_id == processor.tokenizer.eos_token_id:
-            processor.tokenizer.add_special_tokens({'pad_token': '<PAD>'})
-            pad_token_id = processor.tokenizer.convert_tokens_to_ids('<PAD>')
-
-            model.config.pad_token_id = pad_token_id # update model configuration with new pad token ID
-
-        # Tokenize the input batch with the processor
-        inputs = processor(chunk_waveform.squeeze(0).numpy(), sampling_rate=sample_rate, padding="max_length", return_tensors="pt", task="transcribe", device=device)
+        # make sure to NOT truncate the input audio, to return the `attention_mask` and to pad to the longest audio
+        inputs = processor(chunk_waveform.squeeze(0).numpy(), sampling_rate=sample_rate, return_tensors="pt", truncation=False, padding="longest", return_attention_mask=True)
+        inputs = inputs.to("cuda", torch.float32)
 
-        # ASR model inference on chunk
-        with torch.no_grad():
-            generated_ids = model.generate(
-                input_features=inputs.input_features.to(device),
-                **generate_kwargs
-        )
+        input_features = inputs.input_features
+        # transcribe audio to ids
+        generated_ids = model.generate(inputs=input_features,**generate_kwargs)
 
-    # Decode the generated IDs to text
+    # transcription
     chunk_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+#---------------------------------------------------------------------------------------------------------------------------------------------
+#
+#
+#
+#
+#############################################################################################################################################3
     full_text.append(chunk_text)
-
-
     # Combine the transcribed text from all chunks
     text = " ".join(full_text)
 
@@ -132,7 +235,6 @@ def transcribe_audio(audio_file, chunk_length_s=30):
 
     # Audio duration (in seconds)
     audio_duration = waveform.shape[1] / sample_rate
-
     # Real-time Factor (RTF)
     rtf = output_time / audio_duration
 
@@ -146,10 +248,44 @@ def transcribe_audio(audio_file, chunk_length_s=30):
     "It is the ratio of transcription time to the duration of the audio.\n\n"
     "An RTF of less than 1 means the transcription process is faster than real-time (expected)."
     )
+#############################################################################################################################################3
+#
+#
+#
+#
+#---------------------------------------------------------------------------------------------------------------------------------------------
 
     return text, result
-
-
+#---------------------------------------------------------------------------------------------------------------------------------------------
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
 # Clean and preprocess/@summarization
 def clean_text(text):
     text = re.sub(r'https?:\/\/.*[\r\n]*', '', text)
@@ -176,7 +312,7 @@ def summarize_text(text):
     inputs = inputs.to(device)
     summary_ids = summarization_model.generate(inputs.input_ids, num_beams=5, max_length=150, early_stopping=True)
     return summarization_tokenizer.decode(summary_ids[0], skip_special_tokens=True)
-
+requires updating the pre-trained model weights to match
 # Builds similarity matrix
 def build_similarity_matrix(sentences, stop_words):
     similarity_matrix = nx.Graph()
@@ -187,7 +323,7 @@ def build_similarity_matrix(sentences, stop_words):
                 similarity_matrix.add_edge(i, j, weight=len(common_words))
     return similarity_matrix
 
-# "Graph-based summarization" =====> 
+# "Graph-based summarization" =====>
 def graph_based_summary(text, num_paragraphs=3):
     doc = nlp(text)
     sentences = [sent.text for sent in doc.sents]
@@ -262,7 +398,7 @@ iface = gr.Blocks()
 
 PLACEHOLDER = """
 <div style="padding: 30px; text-align: center; display: flex; flex-direction: column; align-items: center;">
-   <img src=""https://huggingface.co/spaces/camparchimedes/ola_s-audioshop/blob/main/pic09w9678yhit.png" alt="" style="width: 100%; height: auto; opacity: 0.93;  "> 
+   <img src=""https://huggingface.co/spaces/camparchimedes/ola_s-audioshop/blob/main/pic09w9678yhit.png" alt="" style="width: 100%; height: auto; opacity: 0.93;  ">
    <h1 style="font-size: 28px; margin-bottom: 2px; opacity: 0.55;">Switch Work | Verktæysett no.1</h1>
    <p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">En webapp for transkribering av lydfiler til norsk skrift. Språkmodell: NbAiLab/nb-whisper-large, Ekstra: oppsummering, pdf-download</p>
 </div>