upgrade to new interface

app.py

@@ -2,31 +2,32 @@ import torch
 import gradio as gr
 import librosa
 
+from espnet2.bin.s2t_inference_language import Speech2Language
 from espnet2.bin.s2t_inference import Speech2Text
-from espnet2.bin.s2t_inference_language import Speech2Text as Speech2Lang
 
 
-TITLE="OWSM: An Open Whisper-style Speech Model from CMU WAVLab"
+TITLE="OWSM: Open Whisper-style Speech Model from CMU WAVLab"
 
 DESCRIPTION='''
-OWSM is an Open Whisper-style Speech Model from [CMU WAVLab](https://www.wavlab.org/).
-It reproduces Whisper-style training using publicly available data and an open-source toolkit [ESPnet](https://github.com/espnet/espnet).
-For more details, please check out our [paper](https://arxiv.org/abs/2309.13876) (Peng et al., ASRU 2023).
+OWSM (pronounced as "awesome") is a series of Open Whisper-style Speech Models from [CMU WAVLab](https://www.wavlab.org/).
+We reproduce Whisper-style training using publicly available data and an open-source toolkit [ESPnet](https://github.com/espnet/espnet).
+For more details, please check our [website](https://www.wavlab.org/activities/2024/owsm/) or [paper](https://arxiv.org/abs/2309.13876) (Peng et al., ASRU 2023).
 
-OWSM v3.1 is an improved version of OWSM v3. It significantly outperforms OWSM v3 in almost all evaluation benchmarks.
-We do not include any new training data. Instead, we utilize a state-of-the-art speech encoder, [E-Branchformer](https://arxiv.org/abs/2210.00077).
+The latest demo uses OWSM v3.1, an improved version of OWSM v3.
+OWSM v3.1 outperforms OWSM v3 in almost all evaluation benchmarks while being faster during inference.
+Note that we do not use extra training data. Instead, we utilize a state-of-the-art speech encoder, [E-Branchformer](https://arxiv.org/abs/2210.00077), to enhance the speech modeling capability.
 
-OWSM v3.1 has 1.02B parameters and is trained on 180k hours of paired speech data. It supports various speech-to-text tasks:
+OWSM v3.1 has 1.02B parameters and is trained on 180k hours of labelled data. It supports various speech-to-text tasks:
 - Speech recognition for 151 languages
 - Any-to-any language speech translation
-- Timestamp prediction
+- Utterance-level timestamp prediction
 - Long-form transcription
 - Language identification
 
 As a demo, the input speech should not exceed 2 minutes. We also limit the maximum number of tokens to be generated.
 Please try our [Colab demo](https://colab.research.google.com/drive/1zKI3ZY_OtZd6YmVeED6Cxy1QwT1mqv9O?usp=sharing) if you want to explore more features.
 
-Disclaimer: OWSM has not been thoroughly evaluated in all tasks. Due to limited training data, it may not perform well for certain language directions.
+Disclaimer: OWSM has not been thoroughly evaluated in all tasks. Due to limited training data, it may not perform well for certain languages.
 
 Please consider citing the following related papers if you find our work helpful.
 
@@ -66,30 +67,29 @@ Please consider citing the following related papers if you find our work helpful
 '''
 
 if not torch.cuda.is_available():
-    raise RuntimeError("Please use GPU for better speed")
+    raise RuntimeError("Please use GPU for better inference speed.")
 
-# model_path = "owsm_v3/exp/s2t_train_s2t_transformer_conv2d_size1024_e24_d24_lr2.5e-4_warmup10k_finetune_raw_bpe50000/valid.acc.ave_5best.till50epoch.pth"
-model_path = "owsm_v3.1_ebf/exp/s2t_train_s2t_ebf_conv2d_size1024_e18_d18_piecewise_lr2e-4_warmup60k_flashattn_raw_bpe50000/valid.total_count.ave_5best.till45epoch.pth"
-device = "cuda" # if torch.cuda.is_available() else "cpu"
+model_tag = "espnet/owsm_v3.1_ebf"
+device = "cuda"
 
-speech2text = Speech2Text.from_pretrained(
-    s2t_model_file=model_path,
+s2l = Speech2Language.from_pretrained(
+    model_tag=model_tag,
     device=device,
-    category_sym="<eng>",
-    beam_size=5,
-    # quantize_s2t_model=not torch.cuda.is_available(),
-    # quantize_dtype="float16",
+    nbest=1,
 )
 
-speech2lang = Speech2Lang.from_pretrained(
-    s2t_model_file=model_path,
+s2t = Speech2Text.from_pretrained(
+    model_tag=model_tag,
     device=device,
-    nbest=1,
-    # quantize_s2t_model=not torch.cuda.is_available(),
-    # quantize_dtype="float16",
+    beam_size=5,
+    ctc_weight=0.0,
+    maxlenratio=0.0,
+    # below are default values which can be overwritten in __call__
+    lang_sym="<eng>",
+    task_sym="<asr>",
+    predict_time=False,
 )
 
-
 iso_codes = ['abk', 'afr', 'amh', 'ara', 'asm', 'ast', 'aze', 'bak', 'bas', 'bel', 'ben', 'bos', 'bre', 'bul', 'cat', 'ceb', 'ces', 'chv', 'ckb', 'cmn', 'cnh', 'cym', 'dan', 'deu', 'dgd', 'div', 'ell', 'eng', 'epo', 'est', 'eus', 'fas', 'fil', 'fin', 'fra', 'frr', 'ful', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hrv', 'hsb', 'hun', 'hye', 'ibo', 'ina', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kam', 'kan', 'kat', 'kaz', 'kea', 'khm', 'kin', 'kir', 'kmr', 'kor', 'lao', 'lav', 'lga', 'lin', 'lit', 'ltz', 'lug', 'luo', 'mal', 'mar', 'mas', 'mdf', 'mhr', 'mkd', 'mlt', 'mon', 'mri', 'mrj', 'mya', 'myv', 'nan', 'nep', 'nld', 'nno', 'nob', 'npi', 'nso', 'nya', 'oci', 'ori', 'orm', 'ory', 'pan', 'pol', 'por', 'pus', 'quy', 'roh', 'ron', 'rus', 'sah', 'sat', 'sin', 'skr', 'slk', 'slv', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'sun', 'swa', 'swe', 'swh', 'tam', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tig', 'tir', 'tok', 'tpi', 'tsn', 'tuk', 'tur', 'twi', 'uig', 'ukr', 'umb', 'urd', 'uzb', 'vie', 'vot', 'wol', 'xho', 'yor', 'yue', 'zho', 'zul']
 lang_names = ['Abkhazian', 'Afrikaans', 'Amharic', 'Arabic', 'Assamese', 'Asturian', 'Azerbaijani', 'Bashkir', 'Basa (Cameroon)', 'Belarusian', 'Bengali', 'Bosnian', 'Breton', 'Bulgarian', 'Catalan', 'Cebuano', 'Czech', 'Chuvash', 'Central Kurdish', 'Mandarin Chinese', 'Hakha Chin', 'Welsh', 'Danish', 'German', 'Dagaari Dioula', 'Dhivehi', 'Modern Greek (1453-)', 'English', 'Esperanto', 'Estonian', 'Basque', 'Persian', 'Filipino', 'Finnish', 'French', 'Northern Frisian', 'Fulah', 'Irish', 'Galician', 'Guarani', 'Gujarati', 'Haitian', 'Hausa', 'Hebrew', 'Hindi', 'Croatian', 'Upper Sorbian', 'Hungarian', 'Armenian', 'Igbo', 'Interlingua (International Auxiliary Language Association)', 'Indonesian', 'Icelandic', 'Italian', 'Javanese', 'Japanese', 'Kabyle', 'Kamba (Kenya)', 'Kannada', 'Georgian', 'Kazakh', 'Kabuverdianu', 'Khmer', 'Kinyarwanda', 'Kirghiz', 'Northern Kurdish', 'Korean', 'Lao', 'Latvian', 'Lungga', 'Lingala', 'Lithuanian', 'Luxembourgish', 'Ganda', 'Luo (Kenya and Tanzania)', 'Malayalam', 'Marathi', 'Masai', 'Moksha', 'Eastern Mari', 'Macedonian', 'Maltese', 'Mongolian', 'Maori', 'Western Mari', 'Burmese', 'Erzya', 'Min Nan Chinese', 'Nepali (macrolanguage)', 'Dutch', 'Norwegian Nynorsk', 'Norwegian Bokmål', 'Nepali (individual language)', 'Pedi', 'Nyanja', 'Occitan (post 1500)', 'Oriya (macrolanguage)', 'Oromo', 'Odia', 'Panjabi', 'Polish', 'Portuguese', 'Pushto', 'Ayacucho Quechua', 'Romansh', 'Romanian', 'Russian', 'Yakut', 'Santali', 'Sinhala', 'Saraiki', 'Slovak', 'Slovenian', 'Shona', 'Sindhi', 'Somali', 'Southern Sotho', 'Spanish', 'Sardinian', 'Serbian', 'Sundanese', 'Swahili (macrolanguage)', 'Swedish', 'Swahili (individual language)', 'Tamil', 'Tatar', 'Telugu', 'Tajik', 'Tagalog', 'Thai', 'Tigre', 'Tigrinya', 'Toki Pona', 'Tok Pisin', 'Tswana', 'Turkmen', 'Turkish', 'Twi', 'Uighur', 'Ukrainian', 'Umbundu', 'Urdu', 'Uzbek', 'Vietnamese', 'Votic', 'Wolof', 'Xhosa', 'Yoruba', 'Yue Chinese', 'Chinese', 'Zulu']
 
@@ -127,34 +127,29 @@ def format_timestamp(
 
 
 def predict(audio_path, src_lang: str, task: str, beam_size, long_form: bool, text_prev: str,):
-    speech2text.task_id = speech2text.converter.token2id[f'<{task2code[task]}>']
-    speech2text.beam_search.beam_size = int(beam_size)
+    task_sym = f'<{task2code[task]}>'
+    s2t.beam_search.beam_size = int(beam_size)
 
     # Our model is trained on 30s and 16kHz
-    _sr = 16000
-    _dur = 30
-    speech, rate = librosa.load(audio_path, sr=_sr) # speech has shape (len,); resample to 16k Hz
+    speech, rate = librosa.load(audio_path, sr=16000) # speech has shape (len,); resample to 16k Hz
 
-    # Detect language using the first 30s of speech
     lang_code = lang2code[src_lang]
     if lang_code == 'none':
-        lang_code = speech2lang(
-            librosa.util.fix_length(speech, size=(_sr * _dur))
-        )[0][0].strip()[1:-1]
-    speech2text.category_id = speech2text.converter.token2id[f'<{lang_code}>']
+        # Detect language using the first 30s of speech
+        lang_code = s2l(speech)[0][0].strip()[1:-1]
+    lang_sym = f'<{lang_code}>'
 
     # ASR or ST
-    if long_form: # speech will be padded in decode_long()
+    if long_form:
         try:
-            speech2text.maxlenratio = -300
-            utts = speech2text.decode_long(
+            s2t.maxlenratio = -300
+            utts = s2t.decode_long(
                 speech,
-                segment_sec=_dur,
-                fs=_sr,
                 condition_on_prev_text=False,
                 init_text=text_prev,
-                start_time="<0.00>",
-                end_time_threshold="<29.50>",
+                end_time_threshold="<29.00>",
+                lang_sym=lang_sym,
+                task_sym=task_sym,
             )
 
             text = []
@@ -166,9 +161,8 @@ def predict(audio_path, src_lang: str, task: str, beam_size, long_form: bool, te
         except:
             print("An exception occurred in long-form decoding. Fall back to standard decoding (only first 30s)")
 
-    speech2text.maxlenratio = -min(300, int((len(speech) / rate) * 10))  # assuming 10 tokens per second
-    speech = librosa.util.fix_length(speech, size=(_sr * _dur))
-    text = speech2text(speech, text_prev)[0][3]
+    s2t.maxlenratio = -min(300, int((len(speech) / rate) * 10))  # assuming 10 tokens per second
+    text = s2t(speech, text_prev, lang_sym=lang_sym, task_sym=task_sym)[0][-2]
 
     return code2lang[lang_code], text
 
@@ -180,7 +174,7 @@ demo = gr.Interface(
         gr.Dropdown(choices=list(lang2code), value="English", label="Language", info="Language of input speech. Select 'Unknown' (1st option) to detect it automatically."),
         gr.Dropdown(choices=list(task2code), value="Automatic Speech Recognition", label="Task", info="Task to perform on input speech."),
         gr.Slider(minimum=1, maximum=5, step=1, value=5, label="Beam Size", info="Beam size used in beam search."),
-        gr.Checkbox(label="Long Form (Experimental)", info="Perform long-form decoding for audios that are longer than 30s. If an exception happens, it will fall back to standard decoding on the initial 30s."),
+        gr.Checkbox(label="Long Form (Experimental)", info="Perform long-form decoding. If an exception happens, it will fall back to standard decoding on the initial 30s."),
         gr.Text(label="Text Prompt (Optional)", info="Generation will be conditioned on this prompt if provided"),
     ],
     outputs=[

owsm_v3.1_ebf/README.md

@@ -1,80 +0,0 @@
----
-tags:
-- espnet
-- audio
-- automatic-speech-recognition
-- speech-translation
-language: multilingual
-datasets:
-- owsm_v3.1
-license: cc-by-4.0
----
-
-## OWSM: Open Whisper-style Speech Model
-
-[OWSM](https://arxiv.org/abs/2309.13876) is an Open Whisper-style Speech Model from [CMU WAVLab](https://www.wavlab.org/). It reproduces Whisper-style training using publicly available data and an open-source toolkit [ESPnet](https://github.com/espnet/espnet).
-
-Our demo is available [here](https://huggingface.co/spaces/pyf98/OWSM_v3_demo).
-
-**OWSM v3.1 is an improved version of OWSM v3. It significantly outperforms OWSM v3 in almost all evaluation benchmarks.**
-We do not include any new training data. Instead, we utilize a state-of-the-art speech encoder, [E-Branchformer](https://arxiv.org/abs/2210.00077).
-
-OWSM v3.1 has 1.02B parameters in total and is trained on 180k hours of public speech data.
-Specifically, it supports the following speech-to-text tasks:
-- Speech recognition
-- Any-to-any-language speech translation
-- Utterance-level alignment
-- Long-form transcription
-- Language identification
-
-
-### Citing OWSM, Branchformers and ESPnet
-
-```BibTex
-@article{peng2023owsm,
-  title={Reproducing Whisper-Style Training Using an Open-Source Toolkit and Publicly Available Data},
-  author={Yifan Peng and Jinchuan Tian and Brian Yan and Dan Berrebbi and Xuankai Chang and Xinjian Li and Jiatong Shi and Siddhant Arora and William Chen and Roshan Sharma and Wangyou Zhang and Yui Sudo and Muhammad Shakeel and Jee-weon Jung and Soumi Maiti and Shinji Watanabe},
-  journal={arXiv preprint arXiv:2309.13876},
-  year={2023}
-}
-@inproceedings{peng23b_interspeech,
-  author={Yifan Peng and Kwangyoun Kim and Felix Wu and Brian Yan and Siddhant Arora and William Chen and Jiyang Tang and Suwon Shon and Prashant Sridhar and Shinji Watanabe},
-  title={{A Comparative Study on E-Branchformer vs Conformer in Speech Recognition, Translation, and Understanding Tasks}},
-  year=2023,
-  booktitle={Proc. INTERSPEECH 2023},
-  pages={2208--2212},
-  doi={10.21437/Interspeech.2023-1194}
-}
-@inproceedings{kim2023branchformer,
-  title={E-branchformer: Branchformer with enhanced merging for speech recognition},
-  author={Kim, Kwangyoun and Wu, Felix and Peng, Yifan and Pan, Jing and Sridhar, Prashant and Han, Kyu J and Watanabe, Shinji},
-  booktitle={2022 IEEE Spoken Language Technology Workshop (SLT)},
-  pages={84--91},
-  year={2023},
-  organization={IEEE}
-}
-@InProceedings{pmlr-v162-peng22a,
-  title = 	 {Branchformer: Parallel {MLP}-Attention Architectures to Capture Local and Global Context for Speech Recognition and Understanding},
-  author =       {Peng, Yifan and Dalmia, Siddharth and Lane, Ian and Watanabe, Shinji},
-  booktitle = 	 {Proceedings of the 39th International Conference on Machine Learning},
-  pages = 	 {17627--17643},
-  year = 	 {2022},
-  editor = 	 {Chaudhuri, Kamalika and Jegelka, Stefanie and Song, Le and Szepesvari, Csaba and Niu, Gang and Sabato, Sivan},
-  volume = 	 {162},
-  series = 	 {Proceedings of Machine Learning Research},
-  month = 	 {17--23 Jul},
-  publisher =    {PMLR},
-  pdf = 	 {https://proceedings.mlr.press/v162/peng22a/peng22a.pdf},
-  url = 	 {https://proceedings.mlr.press/v162/peng22a.html},
-  abstract = 	 {Conformer has proven to be effective in many speech processing tasks. It combines the benefits of extracting local dependencies using convolutions and global dependencies using self-attention. Inspired by this, we propose a more flexible, interpretable and customizable encoder alternative, Branchformer, with parallel branches for modeling various ranged dependencies in end-to-end speech processing. In each encoder layer, one branch employs self-attention or its variant to capture long-range dependencies, while the other branch utilizes an MLP module with convolutional gating (cgMLP) to extract local relationships. We conduct experiments on several speech recognition and spoken language understanding benchmarks. Results show that our model outperforms both Transformer and cgMLP. It also matches with or outperforms state-of-the-art results achieved by Conformer. Furthermore, we show various strategies to reduce computation thanks to the two-branch architecture, including the ability to have variable inference complexity in a single trained model. The weights learned for merging branches indicate how local and global dependencies are utilized in different layers, which benefits model designing.}
-}
-@inproceedings{watanabe2018espnet,
-  author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai},
-  title={{ESPnet}: End-to-End Speech Processing Toolkit},
-  year={2018},
-  booktitle={Proceedings of Interspeech},
-  pages={2207--2211},
-  doi={10.21437/Interspeech.2018-1456},
-  url={http://dx.doi.org/10.21437/Interspeech.2018-1456}
-}
-```

owsm_v3.1_ebf/data/token_list/bpe_unigram50000/bpe.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5d6327da127e870bcb8c737dceb3bd47ccbce63da74ddb094f64afe313d68c8c
-size 1041297

owsm_v3.1_ebf/exp/s2t_stats_raw_bpe50000/train/feats_stats.npz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7ef4b5e465110edf32eec024cf2427eedd677f5733bb87d6b2131e6984a6e13f
-size 1402

owsm_v3.1_ebf/exp/s2t_train_s2t_ebf_conv2d_size1024_e18_d18_piecewise_lr2e-4_warmup60k_flashattn_raw_bpe50000/valid.total_count.ave_5best.till45epoch.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dfb6d34e9f03af6113ada55463d3abe26f133ff2c64e56c65419f9a469313ad3
-size 4068122375

owsm_v3.1_ebf/meta.yaml

@@ -1,8 +0,0 @@
-espnet: '202308'
-files:
-  s2t_model_file: exp/s2t_train_s2t_ebf_conv2d_size1024_e18_d18_piecewise_lr2e-4_warmup60k_flashattn_raw_bpe50000/valid.total_count.ave_5best.till45epoch.pth
-python: 3.10.10 (main, Mar 21 2023, 18:45:11) [GCC 11.2.0]
-timestamp: 1703273348.000399
-torch: 1.13.1
-yaml_files:
-  s2t_train_config: exp/s2t_train_s2t_ebf_conv2d_size1024_e18_d18_piecewise_lr2e-4_warmup60k_flashattn_raw_bpe50000/config.yaml

owsm_v3/data/token_list/bpe_unigram50000/bpe.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:623b8767f80bd60036d8c207b96b25306a8181aa5a702cdac2bf5e90348da174
-size 1042418

owsm_v3/exp/s2t_stats_raw_bpe50000/train/feats_stats.npz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:904a9739b6cdd17afdb4b677627a21d3d1f8ffc99148d8cce07b65395b7e543d
-size 1402

owsm_v3/exp/s2t_train_s2t_transformer_conv2d_size1024_e24_d24_lr2.5e-4_warmup10k_finetune_raw_bpe50000/valid.acc.ave_5best.till50epoch.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0cae90cc63ba655d6e265f60543162cf5a7f8f92205efafeb89547f19403c977
-size 3554533303

requirements.txt

@@ -1,3 +1,3 @@
 torch==2.1.0
 torchaudio
-espnet @ git+https://github.com/espnet/espnet@d3254133c595ea8271072ee49a1b4ceb3ed4fd7a
+espnet @ git+https://github.com/espnet/espnet@7bcb169291f5d4a9b1fd00f8bfe554de84e50024