upload source file

README.md

@@ -1,10 +1,10 @@
 ---
-title: EducaASR
-emoji: 🐠
-colorFrom: green
-colorTo: red
+title: Uetasr
+emoji: 📚
+colorFrom: yellow
+colorTo: blue
 sdk: gradio
-sdk_version: 3.47.1
+sdk_version: 3.24.1
 app_file: app.py
 pinned: false
 ---

app.py

@@ -0,0 +1,299 @@
+import base64
+import gradio as gr
+import librosa
+import logging
+import os
+import soundfile as sf
+import subprocess
+import tempfile
+import urllib.request
+
+from datetime import datetime
+from time import time
+
+from examples import examples
+from model import UETASRModel
+
+
+def get_duration(filename: str) -> float:
+    return librosa.get_duration(path=filename)
+
+
+def convert_to_wav(in_filename: str) -> str:
+    out_filename = os.path.splitext(in_filename)[0] + ".wav"
+    logging.info(f"Converting {in_filename} to {out_filename}")
+    y, sr = librosa.load(in_filename, sr=16000)
+    sf.write(out_filename, y, sr)
+    return out_filename
+
+
+def build_html_output(s: str, style: str = "result_item_success"):
+    return f"""
+    <div class='result'>
+        <div class='result_item {style}'>
+          {s}
+        </div>
+    </div>
+    """
+
+
+def process_url(
+    url: str,
+    decoding_method: str,
+    beam_size: int,
+    max_symbols_per_step: int,
+):
+    logging.info(f"Processing URL: {url}")
+    with tempfile.NamedTemporaryFile() as f:
+        try:
+            urllib.request.urlretrieve(url, f.name)
+            return process(in_filename=f.name,
+                           decoding_method=decoding_method,
+                           beam_size=beam_size,
+                           max_symbols_per_step=max_symbols_per_step)
+        except Exception as e:
+            logging.info(str(e))
+            return "", build_html_output(str(e), "result_item_error")
+
+
+def process_uploaded_file(
+    in_filename: str,
+    decoding_method: str,
+    beam_size: int,
+    max_symbols_per_step: int,
+):
+    if in_filename is None or in_filename == "":
+        return "", build_html_output(
+            "Please first upload a file and then click "
+            'the button "submit for recognition"',
+            "result_item_error",
+        )
+
+    logging.info(f"Processing uploaded file: {in_filename}")
+    try:
+        return process(in_filename=in_filename,
+                       decoding_method=decoding_method,
+                       beam_size=beam_size,
+                       max_symbols_per_step=max_symbols_per_step)
+    except Exception as e:
+        logging.info(str(e))
+        return "", build_html_output(str(e), "result_item_error")
+
+
+def process_microphone(
+    in_filename: str,
+    decoding_method: str,
+    beam_size: int,
+    max_symbols_per_step: int,
+):
+    if in_filename is None or in_filename == "":
+        return "", build_html_output(
+            "Please first upload a file and then click "
+            'the button "submit for recognition"',
+            "result_item_error",
+        )
+
+    logging.info(f"Processing microphone: {in_filename}")
+    try:
+        return process(in_filename=in_filename,
+                       decoding_method=decoding_method,
+                       beam_size=beam_size,
+                       max_symbols_per_step=max_symbols_per_step)
+    except Exception as e:
+        logging.info(str(e))
+        return "", build_html_output(str(e), "result_item_error")
+
+
+def process(
+    in_filename: str,
+    decoding_method: str,
+    beam_size: int,
+    max_symbols_per_step: int,
+):
+    logging.info(f"in_filename: {in_filename}")
+
+    filename = convert_to_wav(in_filename)
+
+    now = datetime.now()
+    date_time = now.strftime("%d/%m/%Y, %H:%M:%S.%f")
+    logging.info(f"Started at {date_time}")
+
+    repo_id = "thanhtvt/uetasr-conformer_30.3m"
+
+    start = time()
+
+    recognizer = UETASRModel(repo_id,
+                             decoding_method,
+                             beam_size,
+                             max_symbols_per_step)
+    text = recognizer.predict(filename)
+
+    date_time = now.strftime("%d/%m/%Y, %H:%M:%S.%f")
+    end = time()
+
+    duration = get_duration(filename)
+    rtf = (end - start) / duration
+
+    logging.info(f"Finished at {date_time} s. Elapsed: {end - start: .3f} s")
+
+    info = f"""
+    Wave duration  : {duration: .3f} s <br/>
+    Processing time: {end - start: .3f} s <br/>
+    RTF: {end - start: .3f}/{duration: .3f} = {rtf:.3f} <br/>
+    """
+    if rtf > 1:
+        info += (
+            "<br/>We are loading required resources for the first run. "
+            "Please run again to measure the real RTF.<br/>"
+        )
+
+    logging.info(info)
+
+    return text, build_html_output(info)
+
+
+title = "Educa ASR"
+description = """
+A space demo for Automatic Speech Recognition.
+"""
+
+# css style is copied from
+# https://huggingface.co/spaces/alphacep/asr/blob/main/app.py#L113
+css = """
+.result {display:flex;flex-direction:column}
+.result_item {padding:15px;margin-bottom:8px;border-radius:15px;width:100%}
+.result_item_success {background-color:mediumaquamarine;color:white;align-self:start}
+.result_item_error {background-color:#ff7070;color:white;align-self:start}
+"""
+
+demo = gr.Blocks(css=css)
+
+
+with demo:
+    gr.Markdown(title)
+
+    decode_method_radio = gr.Radio(
+        label="Decoding method",
+        choices=["greedy_search", "beam_search"],
+        value="greedy_search",
+        interactive=True,
+    )
+
+    beam_size_slider = gr.Slider(
+        label="Beam size",
+        minimum=1,
+        maximum=20,
+        step=1,
+        value=1,
+        interactive=False,
+    )
+
+    def interact_beam_slider(decoding_method):
+        if decoding_method == "greedy_search":
+            return gr.update(value=1, interactive=False)
+        else:
+            return gr.update(interactive=True)
+
+    decode_method_radio.change(interact_beam_slider,
+                               decode_method_radio,
+                               beam_size_slider)
+
+    max_symbols_per_step_slider = gr.Slider(
+        label="Maximum symbols per step",
+        minimum=1,
+        maximum=20,
+        step=1,
+        value=5,
+        interactive=True,
+        visible=True,
+    )
+
+    with gr.Tabs():
+        with gr.TabItem("Upload from disk"):
+            uploaded_file = gr.Audio(
+                source="upload",  # Choose between "microphone", "upload"
+                type="filepath",
+                label="Upload from disk",
+            )
+            upload_button = gr.Button("Submit for recognition")
+            uploaded_output = gr.Textbox(label="Recognized speech from uploaded file")
+            uploaded_html_info = gr.HTML(label="Info")
+
+            gr.Examples(
+                examples=examples,
+                inputs=uploaded_file,
+                outputs=[uploaded_output, uploaded_html_info],
+                fn=process_uploaded_file,
+            )
+
+        with gr.TabItem("Record from microphone"):
+            microphone = gr.Audio(
+                source="microphone",
+                type="filepath",
+                label="Record from microphone",
+            )
+
+            record_button = gr.Button("Submit for recognition")
+            recorded_output = gr.Textbox(label="Recognized speech from recordings")
+            recorded_html_info = gr.HTML(label="Info")
+
+            gr.Examples(
+                examples=examples,
+                inputs=microphone,
+                outputs=[uploaded_output, uploaded_html_info],
+                fn=process_microphone,
+            )
+
+        with gr.TabItem("From URL"):
+            url_textbox = gr.Textbox(
+                max_lines=1,
+                placeholder="URL to an audio file",
+                label="URL",
+                interactive=True,
+            )
+
+            url_button = gr.Button("Submit for recognition")
+            url_output = gr.Textbox(label="Recognized speech from URL")
+            url_html_info = gr.HTML(label="Info")
+
+        upload_button.click(
+            process_uploaded_file,
+            inputs=[
+                uploaded_file,
+                decode_method_radio,
+                beam_size_slider,
+                max_symbols_per_step_slider,
+            ],
+            outputs=[uploaded_output, uploaded_html_info],
+        )
+
+        record_button.click(
+            process_microphone,
+            inputs=[
+                microphone,
+                decode_method_radio,
+                beam_size_slider,
+                max_symbols_per_step_slider,
+            ],
+            outputs=[recorded_output, recorded_html_info],
+        )
+
+        url_button.click(
+            process_url,
+            inputs=[
+                url_textbox,
+                decode_method_radio,
+                beam_size_slider,
+                max_symbols_per_step_slider,
+            ],
+            outputs=[url_output, url_html_info],
+        )
+    gr.Markdown(description)
+
+
+if __name__ == "__main__":
+    formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
+
+    logging.basicConfig(format=formatter, level=logging.INFO)
+
+    demo.launch()

checkpoints/avg_top5_27-32.ckpt.data-00000-of-00001

@@ -0,0 +1 @@
+../../../.cache/huggingface/hub/models--thanhtvt--uetasr-conformer_30.3m/blobs/5806d3e06fac6c2fde5a70f5c5d29bf31dcf81557ef744fd67339804ef736050

checkpoints/avg_top5_27-32.ckpt.index

@@ -0,0 +1 @@
+../../../.cache/huggingface/hub/models--thanhtvt--uetasr-conformer_30.3m/blobs/620561277420e8326531c0036fe013b9f57aa6e577fb7390236679ae15c70ca7

config.yaml

@@ -0,0 +1 @@
+../../.cache/huggingface/hub/models--thanhtvt--uetasr-conformer_30.3m/blobs/665f8dfb04042619a6b114dca8622bd0cfe1ad3d

decode.py

@@ -0,0 +1,37 @@
+import logging
+import tensorflow as tf
+from functools import lru_cache
+from uetasr.searchers import GreedyRNNT, BeamRNNT
+
+
+@lru_cache(maxsize=5)
+def get_searcher(
+    searcher_type: str,
+    decoder: tf.keras.Model,
+    jointer: tf.keras.Model,
+    text_decoder: tf.keras.layers.experimental.preprocessing.PreprocessingLayer,
+    beam_size: int,
+    max_symbols_per_step: int,
+):
+    common_kwargs = {
+        "decoder": decoder,
+        "jointer": jointer,
+        "text_decoder": text_decoder,
+        "return_scores": False,
+    }
+    if searcher_type == "greedy_search":
+        searcher = GreedyRNNT(
+            max_symbols_per_step=max_symbols_per_step,
+            **common_kwargs,
+        )
+    elif searcher_type == "beam_search":
+        searcher = BeamRNNT(
+            max_symbols_per_step=max_symbols_per_step,
+            beam=beam_size,
+            alpha=0.0,
+            **common_kwargs,
+        )
+    else:
+        logging.info(f"Unknown searcher type: {searcher_type}")
+
+    return searcher

examples.py

@@ -0,0 +1,5 @@
+examples = [
+    "./test_wavs/2022_1004_00001300_00002239.wav",
+    "./test_wavs/2022_1004_00087158_00087929.wav",
+    "./test_wavs/2022_1008_00110083_00110571.wav",
+]

model.py

@@ -0,0 +1,135 @@
+import os
+import tensorflow as tf
+from functools import lru_cache
+from huggingface_hub import hf_hub_download
+from hyperpyyaml import load_hyperpyyaml
+from typing import Union
+
+from decode import get_searcher
+
+os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+
+def _get_checkpoint_filename(
+    repo_id: str,
+    filename: str,
+    local_dir: str = None,
+    local_dir_use_symlinks: Union[bool, str] = "auto",
+    subfolder: str = "checkpoints"
+) -> str:
+    model_filename = hf_hub_download(
+        repo_id=repo_id,
+        filename=filename,
+        subfolder=subfolder,
+        local_dir=local_dir,
+        local_dir_use_symlinks=local_dir_use_symlinks,
+    )
+    return model_filename
+
+
+def _get_bpe_model_filename(
+    repo_id: str,
+    filename: str,
+    local_dir: str = None,
+    local_dir_use_symlinks: Union[bool, str] = "auto",
+    subfolder: str = "vocabs"
+) -> str:
+    bpe_model_filename = hf_hub_download(
+        repo_id=repo_id,
+        filename=filename,
+        subfolder=subfolder,
+        local_dir=local_dir,
+        local_dir_use_symlinks=local_dir_use_symlinks,
+    )
+    return bpe_model_filename
+
+
+@lru_cache(maxsize=1)
+def _get_conformer_pre_trained_model(repo_id: str, checkpoint_dir: str = "checkpoints"):
+    for postfix in ["index", "data-00000-of-00001"]:
+        tmp = _get_checkpoint_filename(
+            repo_id=repo_id,
+            filename="avg_top5_27-32.ckpt.{}".format(postfix),
+            subfolder=checkpoint_dir,
+            local_dir=os.path.dirname(__file__),  # noqa
+            local_dir_use_symlinks=True,
+        )
+        print(tmp)
+
+    for postfix in ["model", "vocab"]:
+        tmp = _get_bpe_model_filename(
+            repo_id=repo_id,
+            filename="subword_vietnamese_500.{}".format(postfix),
+            local_dir=os.path.dirname(__file__),  # noqa
+            local_dir_use_symlinks=True,
+        )
+        print(tmp)
+
+    config_path = hf_hub_download(
+        repo_id=repo_id,
+        filename="config.yaml",
+        local_dir=os.path.dirname(__file__),  # noqa
+        local_dir_use_symlinks=True,
+    )
+
+    with open(config_path, "r") as f:
+        config = load_hyperpyyaml(f)
+
+    encoder_model = config["encoder_model"]
+    text_encoder = config["text_encoder"]
+    jointer = config["jointer_model"]
+    decoder = config["decoder_model"]
+    # searcher = config["decoder"]
+    model = config["model"]
+    audio_encoder = config["audio_encoder"]
+    model.load_weights(os.path.join(checkpoint_dir, "avg_top5_27-32.ckpt")).expect_partial()
+
+    return audio_encoder, encoder_model, jointer, decoder, text_encoder, model
+
+
+def read_audio(in_filename: str):
+    audio = tf.io.read_file(in_filename)
+    audio = tf.audio.decode_wav(audio)[0]
+    audio = tf.expand_dims(tf.squeeze(audio, axis=-1), axis=0)
+    return audio
+
+
+class UETASRModel:
+    def __init__(
+        self,
+        repo_id: str,
+        decoding_method: str,
+        beam_size: int,
+        max_symbols_per_step: int,
+    ):
+        self.featurizer, self.encoder_model, jointer, decoder, text_encoder, self.model = _get_conformer_pre_trained_model(repo_id)
+        self.searcher = get_searcher(
+            decoding_method,
+            decoder,
+            jointer,
+            text_encoder,
+            beam_size,
+            max_symbols_per_step,
+        )
+
+    def predict(self, in_filename: str):
+        inputs = read_audio(in_filename)
+        features = self.featurizer(inputs)
+        features = self.model.cmvn(features) if self.model.use_cmvn else features
+
+        mask = tf.sequence_mask([tf.shape(features)[1]], maxlen=tf.shape(features)[1])
+        mask = tf.expand_dims(mask, axis=1)
+        encoder_outputs, encoder_masks = self.encoder_model(
+            features, mask, training=False)
+
+        encoder_mask = tf.squeeze(encoder_masks, axis=1)
+        features_length = tf.math.reduce_sum(
+            tf.cast(encoder_mask, tf.int32),
+            axis=1
+        )
+
+        outputs = self.searcher.infer(encoder_outputs, features_length)
+        outputs = tf.squeeze(outputs)
+        outputs = tf.compat.as_str_any(outputs.numpy())
+
+        return outputs

requirements.txt

@@ -0,0 +1,3 @@
+uetasr @ git+https://github.com/thanhtvt/[email protected]
+requests==2.28.2
+gradio==3.26.0

vocabs/subword_vietnamese_500.model

@@ -0,0 +1 @@
+../../../.cache/huggingface/hub/models--thanhtvt--uetasr-conformer_30.3m/blobs/cd533c4c7981f9d1caeb89be0d0e444177af9e00acf27919d66888625b939b96

vocabs/subword_vietnamese_500.vocab

@@ -0,0 +1 @@
+../../../.cache/huggingface/hub/models--thanhtvt--uetasr-conformer_30.3m/blobs/d6daeedf9cfdad6ec7c7267e23583d2c231d4925