add whisper with language tags and prompt

app.py

@@ -1,34 +1,18 @@
-import torch
 
 import gradio as gr
-import yt_dlp as youtube_dl
-from transformers import pipeline
-from transformers.pipelines.audio_utils import ffmpeg_read
+from whisper2 import generate
 
-import tempfile
-import os
 
-MODEL_NAME = "openai/whisper-large-v3"
-BATCH_SIZE = 8
-FILE_LIMIT_MB = 1000
+MODEL_NAME = "/whisper-large-v3"
 
-device = 0 if torch.cuda.is_available() else "cpu"
 
-pipe = pipeline(
-    task="automatic-speech-recognition",
-    model=MODEL_NAME,
-    chunk_length_s=30,
-    device=device,
-)
 
-def transcribe(inputs, task):
+def transcribe(inputs):
     if inputs is None:
         raise gr.Error("Cap fitxer d'àudio introduit! Si us plau pengeu un fitxer "\
                        "o enregistreu un àudio abans d'enviar la vostra sol·licitud")
 
-
-    text = pipe(inputs, batch_size=BATCH_SIZE, generate_kwargs={"task": task}, return_timestamps=True)["text"]
-    return  text
+    return generate(audio=inputs)
 
 
 description_string = "Transcripció automàtica de micròfon o de fitxers d'àudio.\n Aquest demostrador s'ha desenvolupat per"\
@@ -38,8 +22,7 @@ description_string = "Transcripció automàtica de micròfon o de fitxers d'àud
 file_transcribe = gr.Interface(
     fn=transcribe,
     inputs=[
-        gr.Audio(sources=["upload", "microphone"], type="filepath", label="Audio"),
-        gr.Radio(["transcribe", "translate"], label="Task", value="transcribe"),
+        gr.Audio(sources=["upload", "microphone"], type="filepath", label="Audio")
     ],
     outputs="text",
     title="Transcripció automàtica d'àudio",

requirements.txt

@@ -2,3 +2,5 @@ git+https://github.com/huggingface/transformers
 torch
 yt-dlp
 gradio==4.20.0
+torchaudio==2.2.1
+librosa==0.10.1

whisper2.py

@@ -0,0 +1,103 @@
+from transformers import WhisperForConditionalGeneration, WhisperProcessor
+import torchaudio
+import torch
+import librosa
+
+MODEL_NAME = "openai/whisper-large-v3"
+
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+device = "cpu"
+
+print("[ INFO ] Device: ", device)
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+torch_dtype = torch.float32
+
+model = WhisperForConditionalGeneration.from_pretrained(MODEL_NAME, torch_dtype=torch_dtype).to(device)
+processor = WhisperProcessor.from_pretrained(MODEL_NAME)
+
+
+def convert_forced_to_tokens(forced_decoder_ids):
+    forced_decoder_tokens = []
+    for i, (idx, token) in enumerate(forced_decoder_ids):
+        if token is not None:
+            forced_decoder_tokens.append([idx, processor.tokenizer.decode(token)])
+        else:
+            forced_decoder_tokens.append([idx, token])
+    return forced_decoder_tokens
+
+
+def generate(audio):
+    input_audio, sample_rate = torchaudio.load(audio)
+
+    #metadata = torchaudio.info(audio)
+    #length1 = math.ceil(metadata.num_frames / metadata.sample_rate)
+    length = librosa.get_duration(path=audio)
+
+    input_speech = input_audio[0]
+
+
+    if length <= 30:
+        input_features = processor(input_speech, 
+                                    sampling_rate=16_000, 
+                                    return_tensors="pt", torch_dtype=torch_dtype).input_features.to(device)
+
+    else:
+        input_features = processor(input_speech, 
+                                return_tensors="pt", 
+                                truncation=False, 
+                                padding="longest", 
+                                return_attention_mask=True,
+                                sampling_rate=16_000).input_features.to(device)
+    forced_decoder_ids = []
+    forced_decoder_ids.append([1,50270]) #[1, '<|ca|>']
+    forced_decoder_ids.append([2,50262]) #[2, '<|es|>']
+    forced_decoder_ids.append([3,50360]) #[3, '<|transcribe|>']
+
+    forced_decoder_ids_modified = forced_decoder_ids
+    idx = processor.tokenizer.all_special_tokens.index("<|startofprev|>")
+    forced_bos_token_id = processor.tokenizer.all_special_ids[idx]
+    
+    prompt = " transcribe an audio containing code-switching between es and ca"
+    prompt_tokens = processor.tokenizer(prompt, add_special_tokens=False).input_ids
+
+    # we need to force these tokens
+    forced_decoder_ids = []
+    for idx, token in enumerate(prompt_tokens):
+        # indexing starts from 1 for forced tokens (token at position 0 is the SOS token)
+        forced_decoder_ids.append([idx + 1, token])
+        
+    # now we add the SOS token at the end
+    offset = len(forced_decoder_ids)
+    forced_decoder_ids.append([offset + 1, model.generation_config.decoder_start_token_id])
+
+    # now we need to append the rest of the prefix tokens (lang, task, timestamps)
+    offset = len(forced_decoder_ids)
+    for idx, token in forced_decoder_ids_modified:
+        forced_decoder_ids.append([idx + offset , token])
+
+    model.config.forced_decoder_ids = forced_decoder_ids
+    model.generation_config.forced_decoder_ids = forced_decoder_ids
+
+
+    if length <= 30:
+        pred_ids = model.generate(input_features, 
+                                    return_timestamps=True,
+                                    decoder_start_token_id=forced_bos_token_id,  
+                                    max_new_tokens=128)
+        #exclude prompt from output
+        forced_decoder_tokens = convert_forced_to_tokens(forced_decoder_ids)
+        output = processor.decode(pred_ids[0][len(forced_decoder_tokens) + 1:], skip_special_tokens=True)
+
+    else:
+        pred_ids = model.generate(input_features,
+                                    return_timestamps=True,
+                                    decoder_start_token_id=forced_bos_token_id, 
+                                    logprob_threshold=-1.0, 
+                                    compression_ratio_threshold=1.35,
+                                    temperature=(0.0, 0.2, 0.4),
+                                    no_speech_threshold=0.1,
+                                    )
+        output = processor.batch_decode(pred_ids, skip_special_tokens=True)
+
+    return output[0]