Duplicate from facebook/w2v-bert-2.0

Co-authored-by: Vaibhav Srivastav <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,184 @@
+---
+license: mit
+language:
+- af
+- am
+- ar
+- as
+- az
+- be
+- bn
+- bs
+- bg
+- ca
+- cs
+- zh
+- cy
+- da
+- de
+- el
+- en
+- et
+- fi
+- fr
+- or
+- om
+- ga
+- gl
+- gu
+- ha
+- he
+- hi
+- hr
+- hu
+- hy
+- ig
+- id
+- is
+- it
+- jv
+- ja
+- kn
+- ka
+- kk
+- mn
+- km
+- ky
+- ko
+- lo
+- ln
+- lt
+- lb
+- lg
+- lv
+- ml
+- mr
+- mk
+- mt
+- mi
+- my
+- nl
+- nb
+- ne
+- ny
+- oc
+- pa
+- ps
+- fa
+- pl
+- pt
+- ro
+- ru
+- sk
+- sl
+- sn
+- sd
+- so
+- es
+- sr
+- sv
+- sw
+- ta
+- te
+- tg
+- tl
+- th
+- tr
+- uk
+- ur
+- uz
+- vi
+- wo
+- xh
+- yo
+- ms
+- zu
+- ary
+- arz
+- yue
+- kea
+inference: false
+---
+# W2v-BERT 2.0 speech encoder
+
+We are open-sourcing our Conformer-based [W2v-BERT 2.0 speech encoder](#w2v-bert-20-speech-encoder) as described in Section 3.2.1 of the [paper](https://arxiv.org/pdf/2312.05187.pdf), which is at the core of our Seamless models.
+
+This model was pre-trained on 4.5M hours of unlabeled audio data covering more than 143 languages. It requires finetuning to be used for downstream tasks such as Automatic Speech Recognition (ASR), or Audio Classification.
+
+| Model Name        | #params | checkpoint                                                                                                                                                                                                                                                                                                                                                                 |
+| ----------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| W2v-BERT 2.0 | 600M    | [checkpoint](https://huggingface.co/reach-vb/conformer-shaw/resolve/main/conformer_shaw.pt)
+
+**This model and its training are supported by 🤗 Transformers, more on it in the [docs](https://huggingface.co/docs/transformers/main/en/model_doc/wav2vec2-bert).**
+
+
+# 🤗 Transformers usage
+
+This is a bare checkpoint without any modeling head, and thus requires finetuning to be used for downstream tasks such as ASR. You can however use it to extract audio embeddings from the top layer with this code snippet:
+
+```python
+from transformers import AutoFeatureExtractor, Wav2Vec2BertModel
+import torch
+from datasets import load_dataset
+
+dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation")
+dataset = dataset.sort("id")
+sampling_rate = dataset.features["audio"].sampling_rate
+
+processor = AutoProcessor.from_pretrained("facebook/w2v-bert-2.0")
+model = Wav2Vec2BertModel.from_pretrained("facebook/w2v-bert-2.0")
+
+# audio file is decoded on the fly
+inputs = processor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="pt")
+with torch.no_grad():
+    outputs = model(**inputs)
+```
+
+To learn more about the model use, refer to the following resources:
+- [its docs](https://huggingface.co/docs/transformers/main/en/model_doc/wav2vec2-bert)
+- [a blog post showing how to fine-tune it on Mongolian ASR](https://huggingface.co/blog/fine-tune-w2v2-bert)
+- [a training script example](https://github.com/huggingface/transformers/blob/main/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py)
+
+
+# Seamless Communication usage
+
+This model can be used in [Seamless Communication](https://github.com/facebookresearch/seamless_communication), where it was released.
+ 
+Here's how to make a forward pass through the voice encoder, after having completed the [installation steps](https://github.com/facebookresearch/seamless_communication?tab=readme-ov-file#installation):
+
+```python
+import torch
+
+from fairseq2.data.audio import AudioDecoder, WaveformToFbankConverter
+from fairseq2.memory import MemoryBlock
+from fairseq2.nn.padding import get_seqs_and_padding_mask
+from pathlib import Path
+from seamless_communication.models.conformer_shaw import load_conformer_shaw_model
+
+
+audio_wav_path, device, dtype = ...
+audio_decoder = AudioDecoder(dtype=torch.float32, device=device)
+fbank_converter = WaveformToFbankConverter(
+    num_mel_bins=80,
+    waveform_scale=2**15,
+    channel_last=True,
+    standardize=True,
+    device=device,
+    dtype=dtype,
+)
+collater = Collater(pad_value=1)
+
+model = load_conformer_shaw_model("conformer_shaw", device=device, dtype=dtype)
+model.eval()
+
+with Path(audio_wav_path).open("rb") as fb:
+    block = MemoryBlock(fb.read())
+
+decoded_audio = audio_decoder(block)
+src = collater(fbank_converter(decoded_audio))["fbank"]
+seqs, padding_mask = get_seqs_and_padding_mask(src)
+
+with torch.inference_mode():
+  seqs, padding_mask = model.encoder_frontend(seqs, padding_mask)
+  seqs, padding_mask = model.encoder(seqs, padding_mask)
+```

config.json

@@ -0,0 +1,81 @@
+{
+  "activation_dropout": 0.0,
+  "adapter_act": "relu",
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": false,
+  "architectures": [
+    "Wav2Vec2BertModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "classifier_proj_size": 768,
+  "codevector_dim": 768,
+  "conformer_conv_dropout": 0.1,
+  "contrastive_logits_temperature": 0.1,
+  "conv_depthwise_kernel_size": 31,
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "eos_token_id": 2,
+  "feat_proj_dropout": 0.0,
+  "feat_quantizer_dropout": 0.0,
+  "feature_projection_input_dim": 160,
+  "final_dropout": 0.1,
+  "hidden_act": "swish",
+  "hidden_dropout": 0.0,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.1,
+  "left_max_position_embeddings": 64,
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.05,
+  "max_source_positions": 5000,
+  "model_type": "wav2vec2-bert",
+  "num_adapter_layers": 1,
+  "num_attention_heads": 16,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_hidden_layers": 24,
+  "num_negatives": 100,
+  "output_hidden_size": 1024,
+  "pad_token_id": 0,
+  "position_embeddings_type": "relative_key",
+  "proj_codevector_dim": 768,
+  "right_max_position_embeddings": 8,
+  "rotary_embedding_base": 10000,
+  "tdnn_dilation": [
+    1,
+    2,
+    3,
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    512,
+    512,
+    512,
+    512,
+    1500
+  ],
+  "tdnn_kernel": [
+    5,
+    3,
+    3,
+    1,
+    1
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.0.dev0",
+  "use_intermediate_ffn_before_adapter": false,
+  "use_weighted_layer_sum": false,
+  "vocab_size": null,
+  "xvector_output_dim": 512
+}

conformer_shaw.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8310b4270a5b499e92e20c859892dbf7429619347debb5f8feba79eb88f99b4f
+size 2329131983

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb890c9660ed6e3414b6812e27257b8ce5454365d5490d3ad581ea60b93be043
+size 2322063736

preprocessor_config.json

@@ -0,0 +1,11 @@
+{
+  "feature_extractor_type": "SeamlessM4TFeatureExtractor",
+  "feature_size": 80,
+  "num_mel_bins": 80,
+  "padding_side": "right",
+  "padding_value": 1,
+  "processor_class": "Wav2Vec2BertProcessor",
+  "return_attention_mask": true,
+  "sampling_rate": 16000,
+  "stride": 2
+}