first

app.py

@@ -1,4 +1,65 @@
-import streamlit as st
+import gradio as gr
+import librosa
+import torch
 
-x = st.slider('Select a value')
-st.write(x, 'squared is', x * x)
+from transformers import WhisperProcessor, WhisperForConditionalGeneration
+
+processor = WhisperProcessor.from_pretrained("openai/whisper-large")
+model = SpeechT5ForSpeechToText.from_pretrained("openai/whisper-large")
+
+model.config.forced_decoder_ids = WhisperProcessor.get_decoder_prompt_ids(language="english", task="transcribe")
+
+def process_audio(sampling_rate, waveform):
+    # convert from int16 to floating point
+    waveform = waveform / 32678.0
+
+    # convert to mono if stereo
+    if len(waveform.shape) > 1:
+        waveform = librosa.to_mono(waveform.T)
+
+    # resample to 16 kHz if necessary
+    if sampling_rate != 16000:
+        waveform = librosa.resample(waveform, orig_sr=sampling_rate, target_sr=16000)
+
+    # limit to 30 seconds
+    waveform = waveform[:16000*30]
+
+    # make PyTorch tensor
+    waveform = torch.tensor(waveform)
+    return waveform
+
+def predict(audio, mic_audio=None):
+    # audio = tuple (sample_rate, frames) or (sample_rate, (frames, channels))
+    if mic_audio is not None:
+        sampling_rate, waveform = mic_audio
+    elif audio is not None:
+        sampling_rate, waveform = audio
+    else:
+        return "(please provide audio)"
+
+    waveform = process_audio(sampling_rate, waveform)
+    input_features = processor(waveform, sampling_rate=16000, return_tensors="pt").input_features
+    predicted_ids = model.generate(input_features, max_length=400)
+    transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
+    return transcription[0]
+
+
+title = "Demo for Whisper -> Something -> XLS-R"
+
+description = """
+<b>How to use:</b> Upload an audio file or record using the microphone. The audio is converted to mono and resampled to 16 kHz before
+being passed into the model. The output is the text transcription of the audio.
+"""
+
+gr.Interface(
+    fn=predict,
+    inputs=[
+        gr.Audio(label="Upload Speech", source="upload", type="numpy"),
+        gr.Audio(label="Record Speech", source="microphone", type="numpy"),
+    ],
+    outputs=[
+        gr.Text(label="Transcription"),
+    ],
+    title=title,
+    article=article,
+).launch()