Hugging Face's logo

Spaces:
sanchit-gandhi
/
whisper-jax
Running

App Files Community
176
whisper-jax
File size: 4,596 Bytes 
ee44eab
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e91d345
ee44eab
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
import gradio as gr
import requests
import pytube
from transformers.models.whisper.tokenization_whisper import TO_LANGUAGE_CODE
from transformers.pipelines.audio_utils import ffmpeg_read

title = "Whisper JAX: The Fastest Whisper API Available ⚡️"

description = """Whisper JAX is an optimised implementation of the [Whisper model](https://huggingface.co/openai/whisper-large-v2) by OpenAI. It runs on JAX with a TPU v4-8 in the backend. Compared to PyTorch on an A100 GPU, it is over **12x** faster, making it the fastest Whisper API available.

You can submit requests to Whisper JAX through this Gradio Demo, or directly through API calls (see below). This notebook demonstrates how you can run the Whisper JAX model yourself on a TPU v2-8 in a Google Colab: TODO.
"""

API_URL = "https://whisper-jax.ngrok.io/generate/"

article = "Whisper large-v2 model by OpenAI. Backend running JAX on a TPU v4-8 through the generous support of the [TRC](https://sites.research.google/trc/about/) programme."

language_names = sorted(TO_LANGUAGE_CODE.keys())
SAMPLING_RATE = 16000


def query(payload):
    response = requests.post(API_URL, json=payload)
    return response.json(), response.status_code


def inference(inputs, task, return_timestamps):
    payload = {"inputs": inputs, "task": task, "return_timestamps": return_timestamps}

    data, status_code = query(payload)

    if status_code == 200:
        text = data["text"]
    else:
        text = data["detail"]

    if return_timestamps:
        timestamps = data["chunks"]
    else:
        timestamps = None

    return text, timestamps


def transcribe_audio(microphone, file_upload, task, return_timestamps):
    warn_output = ""
    if (microphone is not None) and (file_upload is not None):
        warn_output = (
            "WARNING: You've uploaded an audio file and used the microphone. "
            "The recorded file from the microphone will be used and the uploaded audio will be discarded.\n"
        )

    elif (microphone is None) and (file_upload is None):
        return "ERROR: You have to either use the microphone or upload an audio file"

    inputs = microphone if microphone is not None else file_upload

    inputs = {"array": inputs[1].tolist(), "sampling_rate": inputs[0]}

    text, timestamps = inference(inputs=inputs, task=task, return_timestamps=return_timestamps)

    return warn_output + text, timestamps


def _return_yt_html_embed(yt_url):
    video_id = yt_url.split("?v=")[-1]
    HTML_str = (
        f'<center> <iframe width="500" height="320" src="https://www.youtube.com/embed/{video_id}"> </iframe>'
        " </center>"
    )
    return HTML_str


def transcribe_youtube(yt_url, task, return_timestamps):
    yt = pytube.YouTube(yt_url)
    html_embed_str = _return_yt_html_embed(yt_url)
    stream = yt.streams.filter(only_audio=True)[0]
    stream.download(filename="audio.mp3")

    with open("audio.mp3", "rb") as f:
        inputs = f.read()

    inputs = ffmpeg_read(inputs, SAMPLING_RATE)
    inputs = {"array": inputs.tolist(), "sampling_rate": SAMPLING_RATE}

    yield html_embed_str, "Video loaded, transcribing audio...", None

    text, timestamps = inference(inputs=inputs, task=task, return_timestamps=return_timestamps)

    yield html_embed_str, text, timestamps

audio = gr.Interface(
    fn=transcribe_audio,
    inputs=[
        gr.inputs.Audio(source="microphone", optional=True),
        gr.inputs.Audio(source="upload", optional=True),
        gr.inputs.Radio(["transcribe", "translate"], label="Task", default="transcribe"),
        gr.inputs.Checkbox(default=False, label="Return timestamps"),
    ],
    outputs=[
        gr.outputs.Textbox(label="Transcription"),
        gr.outputs.Textbox(label="Timestamps"),
    ],
    allow_flagging="never",
    title=title,
    description=description,
    article=article,
)

youtube = gr.Interface(
    fn=transcribe_youtube,
    inputs=[
        gr.inputs.Textbox(lines=1, placeholder="Paste the URL to a YouTube video here", label="YouTube URL"),
        gr.inputs.Radio(["transcribe", "translate"], label="Task", default="transcribe"),
        gr.inputs.Checkbox(default=False, label="Return timestamps"),
    ],
    outputs=[
        gr.outputs.HTML(label="Video"),
        gr.outputs.Textbox(label="Transcription"),
        gr.outputs.Textbox(label="Timestamps"),
    ],
    allow_flagging="never",
    title=title,
    description=description,
    article=article,
)

demo = gr.Blocks()

with demo:
    gr.TabbedInterface([audio, youtube], ["Transcribe Audio", "Transcribe YouTube"])

demo.queue()
demo.launch()