Update README.md

README.md

@@ -1,245 +1,243 @@
----
-language:
-- ace
-- acm
-- acq
-- aeb
-- af
-- ajp
-- ak
-- als
-- am
-- apc
-- ar
-- ars
-- ary
-- arz
-- as
-- ast
-- awa
-- ayr
-- azb
-- azj
-- ba
-- bm
-- ban
-- be
-- bem
-- bn
-- bho
-- bjn
-- bo
-- bs
-- bug
-- bg
-- ca
-- ceb
-- cs
-- cjk
-- ckb
-- crh
-- cy
-- da
-- de
-- dik
-- dyu
-- dz
-- el
-- en
-- eo
-- et
-- eu
-- ee
-- fo
-- fj
-- fi
-- fon
-- fr
-- fur
-- fuv
-- gaz
-- gd
-- ga
-- gl
-- gn
-- gu
-- ht
-- ha
-- he
-- hi
-- hne
-- hr
-- hu
-- hy
-- ig
-- ilo
-- id
-- is
-- it
-- jv
-- ja
-- kab
-- kac
-- kam
-- kn
-- ks
-- ka
-- kk
-- kbp
-- kea
-- khk
-- km
-- ki
-- rw
-- ky
-- kmb
-- kmr
-- knc
-- kg
-- ko
-- lo
-- lij
-- li
-- ln
-- lt
-- lmo
-- ltg
-- lb
-- lua
-- lg
-- luo
-- lus
-- lvs
-- mag
-- mai
-- ml
-- mar
-- min
-- mk
-- mt
-- mni
-- mos
-- mi
-- my
-- nl
-- nn
-- nb
-- npi
-- nso
-- nus
-- ny
-- oc
-- ory
-- pag
-- pa
-- pap
-- pbt
-- pes
-- plt
-- pl
-- pt
-- prs
-- quy
-- ro
-- rn
-- ru
-- sg
-- sa
-- sat
-- scn
-- shn
-- si
-- sk
-- sl
-- sm
-- sn
-- sd
-- so
-- st
-- es
-- sc
-- sr
-- ss
-- su
-- sv
-- swh
-- szl
-- ta
-- taq
-- tt
-- te
-- tg
-- tl
-- th
-- ti
-- tpi
-- tn
-- ts
-- tk
-- tum
-- tr
-- tw
-- tzm
-- ug
-- uk
-- umb
-- ur
-- uzn
-- vec
-- vi
-- war
-- wo
-- xh
-- ydd
-- yo
-- yue
-- zh
-- zsm
-- zu
-language_details: >-
-  ace_Arab, ace_Latn, acm_Arab, acq_Arab, aeb_Arab, afr_Latn, ajp_Arab,
-  aka_Latn, amh_Ethi, apc_Arab, arb_Arab, ars_Arab, ary_Arab, arz_Arab,
-  asm_Beng, ast_Latn, awa_Deva, ayr_Latn, azb_Arab, azj_Latn, bak_Cyrl,
-  bam_Latn, ban_Latn,bel_Cyrl, bem_Latn, ben_Beng, bho_Deva, bjn_Arab, bjn_Latn,
-  bod_Tibt, bos_Latn, bug_Latn, bul_Cyrl, cat_Latn, ceb_Latn, ces_Latn,
-  cjk_Latn, ckb_Arab, crh_Latn, cym_Latn, dan_Latn, deu_Latn, dik_Latn,
-  dyu_Latn, dzo_Tibt, ell_Grek, eng_Latn, epo_Latn, est_Latn, eus_Latn,
-  ewe_Latn, fao_Latn, pes_Arab, fij_Latn, fin_Latn, fon_Latn, fra_Latn,
-  fur_Latn, fuv_Latn, gla_Latn, gle_Latn, glg_Latn, grn_Latn, guj_Gujr,
-  hat_Latn, hau_Latn, heb_Hebr, hin_Deva, hne_Deva, hrv_Latn, hun_Latn,
-  hye_Armn, ibo_Latn, ilo_Latn, ind_Latn, isl_Latn, ita_Latn, jav_Latn,
-  jpn_Jpan, kab_Latn, kac_Latn, kam_Latn, kan_Knda, kas_Arab, kas_Deva,
-  kat_Geor, knc_Arab, knc_Latn, kaz_Cyrl, kbp_Latn, kea_Latn, khm_Khmr,
-  kik_Latn, kin_Latn, kir_Cyrl, kmb_Latn, kon_Latn, kor_Hang, kmr_Latn,
-  lao_Laoo, lvs_Latn, lij_Latn, lim_Latn, lin_Latn, lit_Latn, lmo_Latn,
-  ltg_Latn, ltz_Latn, lua_Latn, lug_Latn, luo_Latn, lus_Latn, mag_Deva,
-  mai_Deva, mal_Mlym, mar_Deva, min_Latn, mkd_Cyrl, plt_Latn, mlt_Latn,
-  mni_Beng, khk_Cyrl, mos_Latn, mri_Latn, zsm_Latn, mya_Mymr, nld_Latn,
-  nno_Latn, nob_Latn, npi_Deva, nso_Latn, nus_Latn, nya_Latn, oci_Latn,
-  gaz_Latn, ory_Orya, pag_Latn, pan_Guru, pap_Latn, pol_Latn, por_Latn,
-  prs_Arab, pbt_Arab, quy_Latn, ron_Latn, run_Latn, rus_Cyrl, sag_Latn,
-  san_Deva, sat_Beng, scn_Latn, shn_Mymr, sin_Sinh, slk_Latn, slv_Latn,
-  smo_Latn, sna_Latn, snd_Arab, som_Latn, sot_Latn, spa_Latn, als_Latn,
-  srd_Latn, srp_Cyrl, ssw_Latn, sun_Latn, swe_Latn, swh_Latn, szl_Latn,
-  tam_Taml, tat_Cyrl, tel_Telu, tgk_Cyrl, tgl_Latn, tha_Thai, tir_Ethi,
-  taq_Latn, taq_Tfng, tpi_Latn, tsn_Latn, tso_Latn, tuk_Latn, tum_Latn,
-  tur_Latn, twi_Latn, tzm_Tfng, uig_Arab, ukr_Cyrl, umb_Latn, urd_Arab,
-  uzn_Latn, vec_Latn, vie_Latn, war_Latn, wol_Latn, xho_Latn, ydd_Hebr,
-  yor_Latn, yue_Hant, zho_Hans, zho_Hant, zul_Latn
-license: mit
-metrics:
-- bleu
-datasets:
-- mozilla-foundation/common_voice_8_0
-pipeline_tag: automatic-speech-recognition
-tags:
-- zeroswot
-- speech translation
-- zero-shot
-- end-to-end
-- nllb
-- wav2vec2
----
+---
+language:
+- ace
+- acm
+- acq
+- aeb
+- af
+- ajp
+- ak
+- als
+- am
+- apc
+- ar
+- ars
+- ary
+- arz
+- as
+- ast
+- awa
+- ayr
+- azb
+- azj
+- ba
+- bm
+- ban
+- be
+- bem
+- bn
+- bho
+- bjn
+- bo
+- bs
+- bug
+- bg
+- ca
+- ceb
+- cs
+- cjk
+- ckb
+- crh
+- cy
+- da
+- de
+- dik
+- dyu
+- dz
+- el
+- en
+- eo
+- et
+- eu
+- ee
+- fo
+- fj
+- fi
+- fon
+- fr
+- fur
+- fuv
+- gaz
+- gd
+- ga
+- gl
+- gn
+- gu
+- ht
+- ha
+- he
+- hi
+- hne
+- hr
+- hu
+- hy
+- ig
+- ilo
+- id
+- is
+- it
+- jv
+- ja
+- kab
+- kac
+- kam
+- kn
+- ks
+- ka
+- kk
+- kbp
+- kea
+- khk
+- km
+- ki
+- rw
+- ky
+- kmb
+- kmr
+- knc
+- kg
+- ko
+- lo
+- lij
+- li
+- ln
+- lt
+- lmo
+- ltg
+- lb
+- lua
+- lg
+- luo
+- lus
+- lvs
+- mag
+- mai
+- ml
+- mar
+- min
+- mk
+- mt
+- mni
+- mos
+- mi
+- my
+- nl
+- nn
+- nb
+- npi
+- nso
+- nus
+- ny
+- oc
+- ory
+- pag
+- pa
+- pap
+- pbt
+- pes
+- plt
+- pl
+- pt
+- prs
+- quy
+- ro
+- rn
+- ru
+- sg
+- sa
+- sat
+- scn
+- shn
+- si
+- sk
+- sl
+- sm
+- sn
+- sd
+- so
+- st
+- es
+- sc
+- sr
+- ss
+- su
+- sv
+- swh
+- szl
+- ta
+- taq
+- tt
+- te
+- tg
+- tl
+- th
+- ti
+- tpi
+- tn
+- ts
+- tk
+- tum
+- tr
+- tw
+- tzm
+- ug
+- uk
+- umb
+- ur
+- uzn
+- vec
+- vi
+- war
+- wo
+- xh
+- ydd
+- yo
+- yue
+- zh
+- zsm
+- zu
+language_details: >-
+  ace_Arab, ace_Latn, acm_Arab, acq_Arab, aeb_Arab, afr_Latn, ajp_Arab,
+  aka_Latn, amh_Ethi, apc_Arab, arb_Arab, ars_Arab, ary_Arab, arz_Arab,
+  asm_Beng, ast_Latn, awa_Deva, ayr_Latn, azb_Arab, azj_Latn, bak_Cyrl,
+  bam_Latn, ban_Latn,bel_Cyrl, bem_Latn, ben_Beng, bho_Deva, bjn_Arab, bjn_Latn,
+  bod_Tibt, bos_Latn, bug_Latn, bul_Cyrl, cat_Latn, ceb_Latn, ces_Latn,
+  cjk_Latn, ckb_Arab, crh_Latn, cym_Latn, dan_Latn, deu_Latn, dik_Latn,
+  dyu_Latn, dzo_Tibt, ell_Grek, eng_Latn, epo_Latn, est_Latn, eus_Latn,
+  ewe_Latn, fao_Latn, pes_Arab, fij_Latn, fin_Latn, fon_Latn, fra_Latn,
+  fur_Latn, fuv_Latn, gla_Latn, gle_Latn, glg_Latn, grn_Latn, guj_Gujr,
+  hat_Latn, hau_Latn, heb_Hebr, hin_Deva, hne_Deva, hrv_Latn, hun_Latn,
+  hye_Armn, ibo_Latn, ilo_Latn, ind_Latn, isl_Latn, ita_Latn, jav_Latn,
+  jpn_Jpan, kab_Latn, kac_Latn, kam_Latn, kan_Knda, kas_Arab, kas_Deva,
+  kat_Geor, knc_Arab, knc_Latn, kaz_Cyrl, kbp_Latn, kea_Latn, khm_Khmr,
+  kik_Latn, kin_Latn, kir_Cyrl, kmb_Latn, kon_Latn, kor_Hang, kmr_Latn,
+  lao_Laoo, lvs_Latn, lij_Latn, lim_Latn, lin_Latn, lit_Latn, lmo_Latn,
+  ltg_Latn, ltz_Latn, lua_Latn, lug_Latn, luo_Latn, lus_Latn, mag_Deva,
+  mai_Deva, mal_Mlym, mar_Deva, min_Latn, mkd_Cyrl, plt_Latn, mlt_Latn,
+  mni_Beng, khk_Cyrl, mos_Latn, mri_Latn, zsm_Latn, mya_Mymr, nld_Latn,
+  nno_Latn, nob_Latn, npi_Deva, nso_Latn, nus_Latn, nya_Latn, oci_Latn,
+  gaz_Latn, ory_Orya, pag_Latn, pan_Guru, pap_Latn, pol_Latn, por_Latn,
+  prs_Arab, pbt_Arab, quy_Latn, ron_Latn, run_Latn, rus_Cyrl, sag_Latn,
+  san_Deva, sat_Beng, scn_Latn, shn_Mymr, sin_Sinh, slk_Latn, slv_Latn,
+  smo_Latn, sna_Latn, snd_Arab, som_Latn, sot_Latn, spa_Latn, als_Latn,
+  srd_Latn, srp_Cyrl, ssw_Latn, sun_Latn, swe_Latn, swh_Latn, szl_Latn,
+  tam_Taml, tat_Cyrl, tel_Telu, tgk_Cyrl, tgl_Latn, tha_Thai, tir_Ethi,
+  taq_Latn, taq_Tfng, tpi_Latn, tsn_Latn, tso_Latn, tuk_Latn, tum_Latn,
+  tur_Latn, twi_Latn, tzm_Tfng, uig_Arab, ukr_Cyrl, umb_Latn, urd_Arab,
+  uzn_Latn, vec_Latn, vie_Latn, war_Latn, wol_Latn, xho_Latn, ydd_Hebr,
+  yor_Latn, yue_Hant, zho_Hans, zho_Hant, zul_Latn
+license: mit
+metrics:
+- bleu
+pipeline_tag: automatic-speech-recognition
+tags:
+- zeroswot
+- speech translation
+- zero-shot
+- end-to-end
+- nllb
+- wav2vec2
+---
 
 # ZeroSwot ✨🤖✨
 
@@ -265,7 +263,7 @@ The compression module is a light-weight transformer that takes as input the hid
 
 ## Version
 
-This version of ZeroSwot is trained with ASR data from CommonVoice, and adapted [wav2vec2.0-large](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self) to the [nllb-200-distilled-600M](https://huggingface.co/facebook/nllb-200-distilled-600M) model.
+This version of ZeroSwot is trained with ASR data from MuST-C v1.0, and adapted [wav2vec2.0-large](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self) to the [nllb-200-distilled-1.3B](https://huggingface.co/facebook/nllb-200-distilled-1.3B) model.
 
 We have more versions available:
 
@@ -302,18 +300,18 @@ def load_and_resample_audio(audio_path, target_sr=16000):
 
 # Load processors and tokenizers
 processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-large-960h-lv60-self")
-tokenizer = NllbTokenizer.from_pretrained("facebook/nllb-200-distilled-600M")
+tokenizer = NllbTokenizer.from_pretrained("facebook/nllb-200-distilled-1.3B")
 
 # Load ZeroSwot Encoder
-commit_hash = "eafabee295ea1c8b45483d1fd26bd747d9a7d937"
+commit_hash = "9cd290f5eef6dc179819815e7c970ae73616f9fa"
 zeroswot_encoder = AutoModel.from_pretrained(
-    "johntsi/ZeroSwot-Medium_asr-cv_en-to-200", trust_remote_code=True, revision=commit_hash,
+    "johntsi/ZeroSwot-Large_asr-mustc_en-to-200", trust_remote_code=True, revision=commit_hash,
 )
 zeroswot_encoder.eval()
 zeroswot_encoder.to("cuda")
 
 # Load NLLB Model
-nllb_model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M")
+nllb_model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-1.3B")
 nllb_model.eval()
 nllb_model.to("cuda")
 
@@ -335,14 +333,24 @@ print(translation)
 
 ## Results
 
-BLEU scores on CoVoST-2 test compared to supervised SOTA models [XLS-R-1B](https://huggingface.co/facebook/wav2vec2-xls-r-1b) and [SeamlessM4T-Medium](https://huggingface.co/facebook/seamless-m4t-medium). You can refer to Table 5 of the Results section in the paper for more details.
+BLEU scores on MuST-C v1.0 tst-COMMON compared to _supervised_ SOTA models from the literature. You can refer to Table 5 of the Results section in the paper for more details.
 
-|     Models     |  ZS  |  Size (B)  |  Ar  |  Ca  |  Cy  |  De  |  Et  |  Fa  |  Id  |  Ja  |  Lv  |  Mn  |  Sl  |  Sv  |  Ta  |  Tr  |  Zh  | Average |
-|:--------------:|:----:|:----------:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:-------:|
-|    [XLS-R-1B](https://huggingface.co/facebook/wav2vec2-xls-r-1b)    |  ✗   |    1.0     | 19.2 | 32.1 | **31.8** | 26.2 | 22.4 | 21.3 | 30.3 | 39.9 | 22.0 | 14.9 | 25.4 | 32.3 | 18.1 | 17.1 | 36.7 |   26.0  |
-| [SeamlessM4T-Medium](https://huggingface.co/facebook/seamless-m4t-medium)  |  ✗   |    1.2     | 20.8 | 37.3 | 29.9 | **31.4** | 23.3 | 17.2 | 34.8 | 37.5 | 19.5 | 12.9 | 29.0 | 37.3 | 18.9 | **19.8** | 30.0 |   26.6  |
-| [ZeroSwot-M_asr-cv](https://huggingface.co/johntsi/ZeroSwot-Medium_asr-cv_en-to-200) |  ✓   | 0.35/0.95  | 17.6 | 32.5 | 18.0 | 29.9 | 20.4 | 16.3 | 32.4 | 32.0 | 13.3 | 10.0 | 25.2 | 34.4 | 17.8 | 15.6 | 30.5 |   23.1  |
-| [ZeroSwot-M_asr-cv_mt-covost2](https://huggingface.co/johntsi/ZeroSwot-Medium_asr-cv_mt-covost2_en-to-200) |  ✓   | 0.35/0.95  | **24.4** | **38.7** | 28.8 | 31.2 | **26.2** | **26.0** | **36.0** | **46.0** | **24.8** | **19.0** | **31.6** | **37.8** | **24.4** | 18.6 | **39.0** |   **30.2**  |
+|          Models         |  ZS  |  Size (B)  |  De  |  Es  |  Fr  |  It  |  Nl  |  Pt  |  Ro  |  Ru  | Average |
+|:-----------------------:|:----:|:----------:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:-------:|
+| Chimera (Han et al., 2021)           | ✗ | 0.15      | 27.1 | 30.6 | 35.6 | 25.0 | 29.2 | 30.2 | 24.0 | 17.4 | 27.4 |
+| STEMM (Fang et al., 2022)            | ✗ | 0.15    | 28.7 | 31.0 | 37.4 | 25.8 | 30.5 | 31.7 | 24.5 | 17.8 | 28.4 |
+| SpeechUT (Zhang et al., 2022)        | ✗ | 0.15     | 30.1 | 33.6 | 41.4 | -    | -    | -    | -    | -    | -    |
+| Siamese-PT (Le et al., 2023)         | ✗ | 0.25    | 27.9 | 31.8 | 39.2 | 27.7 | 31.7 | 34.2 | 27.0 | 18.5 | 29.8 |
+| CRESS (Fang and Feng, 2023)          | ✗ | 0.15    | 29.4 | 33.2 | 40.1 | 27.6 | 32.2 | 33.6 | 26.4 | 19.7 | 30.3 |
+| SimRegCR (Gao et al., 2023b)         | ✗ | 0.15    | 29.2 | 33.0 | 40.0 | 28.2 | 32.7 | 34.2 | 26.7 | 20.1 | 30.5 |
+| LST (LLaMA2-13B) (Zhang et al., 2023)| ✗ | 13       | 30.4 | 35.3 | **41.6** | -    | -    | -    | -    | -    | -    |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| [ZeroSwot-Medium_asr-cv](https://huggingface.co/johntsi/ZeroSwot-Medium_asr-cv_en-to-200)                            | ✓ | 0.35/0.95 | 24.8 | 30.0 | 32.6 | 24.1 | 28.6 | 28.8 | 22.9 | 16.4 | 26.0 |
+| [ZeroSwot-Medium_asr-mustc](https://huggingface.co/johntsi/ZeroSwot-Medium_asr-mustc_en-to-200)                 | ✓ | 0.35/0.95 | 28.5 | 33.1 | 37.5 | 28.2 | 32.3 | 32.9 | 26.0 | 18.7 | 29.6 |
+| [ZeroSwot-Medium_asr-mustc_mt-mustc](https://huggingface.co/johntsi/ZeroSwot-Medium_asr-mustc_mt-mustc_en-to-8)    | ✓ | 0.35/0.95†| 30.5 | 34.9 | 39.4 | 30.6 | 35.0 | 37.1 | 27.8 | 20.3 | 31.9 |
+| [ZeroSwot-Large_asr-cv](https://huggingface.co/johntsi/ZeroSwot-Large_asr-cv_en-to-200)                                 | ✓ | 0.35/1.65 | 26.5 | 31.1 | 33.5 | 25.4 | 29.9 | 30.6 | 24.3 | 18.0 | 27.4 |
+| [ZeroSwot-Large_asr-mustc](https://huggingface.co/johntsi/ZeroSwot-Large_asr-mustc_en-to-200)| ✓ | 0.35/1.65 | 30.1 | 34.8 | 38.9 | 29.8 | 34.4 | 35.3 | 27.6 | 20.4 | 31.4 |
+| [ZeroSwot-Large_asr-mustc_mt-mustc](https://huggingface.co/johntsi/ZeroSwot-Large_asr-mustc_mt-mustc_en-to-8)| ✓ | 0.35/1.65†| **31.2** | **35.8** | 40.5 | **31.4** | **36.3** | **38.3** | **28.0** | **21.5** | **32.9** |
 
 ## Citation