"
]
},
"execution_count": 51,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import IPython.display as ipd\n",
"import numpy as np\n",
"import random\n",
"\n",
"rand_int = random.randint(0, len(common_voice_train)-1)\n",
"\n",
"print(\"Target text:\", common_voice_train[rand_int][\"sentence\"])\n",
"print(\"Input array shape:\", common_voice_train[rand_int][\"audio\"][\"array\"].shape)\n",
"print(\"Sampling rate:\", common_voice_train[rand_int][\"audio\"][\"sampling_rate\"])\n",
"ipd.Audio(data=common_voice_train[rand_int][\"audio\"][\"array\"], autoplay=True, rate=16000)"
]
},
{
"cell_type": "code",
"execution_count": 52,
"id": "1cb97a7e",
"metadata": {},
"outputs": [],
"source": [
"# This does not prepare the input for the Transformer model.\n",
"# This will resample the data and convert the sentence into indices\n",
"# Batch here is just for one entry (row)\n",
"def prepare_dataset(batch):\n",
" audio = batch[\"audio\"]\n",
" \n",
" # batched output is \"un-batched\"\n",
" batch[\"input_values\"] = processor(audio[\"array\"], sampling_rate=audio[\"sampling_rate\"]).input_values[0]\n",
" batch[\"input_length\"] = len(batch[\"input_values\"])\n",
" \n",
" with processor.as_target_processor():\n",
" batch[\"labels\"] = processor(batch[\"sentence\"]).input_ids\n",
" return batch"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "d7bd14b2",
"metadata": {},
"outputs": [],
"source": [
"common_voice_train = common_voice_train.map(prepare_dataset, remove_columns=common_voice_train.column_names, num_proc=16)\n",
"common_voice_test = common_voice_test.map(prepare_dataset, remove_columns=common_voice_test.column_names, num_proc=16)"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "00c75e3c",
"metadata": {},
"outputs": [],
"source": [
"# In case the dataset is too long which can lead to OOM. We should filter them out.\n",
"# max_input_length_in_sec = 5.0\n",
"# common_voice_train = common_voice_train.filter(lambda x: x < max_input_length_in_sec * processor.feature_extractor.sampling_rate, input_columns=[\"input_length\"])"
]
},
{
"cell_type": "code",
"execution_count": 54,
"id": "8906918e",
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"\n",
"from dataclasses import dataclass, field\n",
"from typing import Any, Dict, List, Optional, Union\n",
"\n",
"@dataclass\n",
"class DataCollatorCTCWithPadding:\n",
" \"\"\"\n",
" Data collator that will dynamically pad the inputs received.\n",
" Args:\n",
" processor (:class:`~transformers.Wav2Vec2Processor`)\n",
" The processor used for proccessing the data.\n",
" padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):\n",
" Select a strategy to pad the returned sequences (according to the model's padding side and padding index)\n",
" among:\n",
" * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single\n",
" sequence if provided).\n",
" * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the\n",
" maximum acceptable input length for the model if that argument is not provided.\n",
" * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of\n",
" different lengths).\n",
" \"\"\"\n",
"\n",
" processor: Wav2Vec2Processor\n",
" padding: Union[bool, str] = True\n",
"\n",
" def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:\n",
" # split inputs and labels since they have to be of different lenghts and need\n",
" # different padding methods\n",
" input_features = [{\"input_values\": feature[\"input_values\"]} for feature in features]\n",
" label_features = [{\"input_ids\": feature[\"labels\"]} for feature in features]\n",
"\n",
" batch = self.processor.pad(\n",
" input_features,\n",
" padding=self.padding,\n",
" return_tensors=\"pt\",\n",
" )\n",
" with self.processor.as_target_processor():\n",
" labels_batch = self.processor.pad(\n",
" label_features,\n",
" padding=self.padding,\n",
" return_tensors=\"pt\",\n",
" )\n",
"\n",
" # replace padding with -100 to ignore loss correctly\n",
" labels = labels_batch[\"input_ids\"].masked_fill(labels_batch.attention_mask.ne(1), -100)\n",
"\n",
" batch[\"labels\"] = labels\n",
"\n",
" return batch"
]
},
{
"cell_type": "code",
"execution_count": 55,
"id": "40ff3940",
"metadata": {},
"outputs": [],
"source": [
"data_collator = DataCollatorCTCWithPadding(processor=processor, padding=True)"
]
},
{
"cell_type": "code",
"execution_count": 56,
"id": "2a579f5a",
"metadata": {},
"outputs": [],
"source": [
"# wer_metric = load_metric(\"wer\")\n",
"cer_metric = load_metric(\"cer\")"
]
},
{
"cell_type": "code",
"execution_count": 57,
"id": "221d53c7",
"metadata": {},
"outputs": [],
"source": [
"def compute_metrics(pred):\n",
" pred_logits = pred.predictions\n",
" pred_ids = np.argmax(pred_logits, axis=-1)\n",
"\n",
" pred.label_ids[pred.label_ids == -100] = processor.tokenizer.pad_token_id\n",
"\n",
" pred_str = processor.batch_decode(pred_ids)\n",
" # we do not want to group tokens when computing the metrics\n",
" label_str = processor.batch_decode(pred.label_ids, group_tokens=False)\n",
"\n",
" # wer = wer_metric.compute(predictions=pred_str, references=label_str)\n",
" cer = cer_metric.compute(predictions=pred_str, references=label_str)\n",
"\n",
" # return {\"wer\": wer}\n",
" return {\"cer\": cer}"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "87cf1d87",
"metadata": {},
"outputs": [],
"source": [
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d8afce23",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 58,
"id": "22a21f3f",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Some weights of the model checkpoint at facebook/wav2vec2-xls-r-300m were not used when initializing Wav2Vec2ForCTC: ['project_hid.bias', 'quantizer.weight_proj.weight', 'quantizer.weight_proj.bias', 'project_q.weight', 'project_hid.weight', 'quantizer.codevectors', 'project_q.bias']\n",
"- This IS expected if you are initializing Wav2Vec2ForCTC from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
"- This IS NOT expected if you are initializing Wav2Vec2ForCTC from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
"Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at facebook/wav2vec2-xls-r-300m and are newly initialized: ['lm_head.weight', 'lm_head.bias']\n",
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
]
}
],
"source": [
"from transformers import Wav2Vec2ForCTC\n",
"\n",
"model = Wav2Vec2ForCTC.from_pretrained(\n",
" \"facebook/wav2vec2-xls-r-300m\", \n",
" attention_dropout=0.0,\n",
" hidden_dropout=0.0,\n",
" feat_proj_dropout=0.0,\n",
" mask_time_prob=0.05,\n",
" layerdrop=0.0,\n",
" ctc_loss_reduction=\"mean\", \n",
" pad_token_id=processor.tokenizer.pad_token_id,\n",
" vocab_size=len(processor.tokenizer),\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 59,
"id": "1688901e",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/lib/python3.8/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py:1700: FutureWarning: The method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5.Please use the equivalent `freeze_feature_encoder` method instead.\n",
" warnings.warn(\n"
]
}
],
"source": [
"model.freeze_feature_extractor()"
]
},
{
"cell_type": "code",
"execution_count": 60,
"id": "e13830f8",
"metadata": {},
"outputs": [],
"source": [
"from transformers import TrainingArguments\n",
"\n",
"training_args = TrainingArguments(\n",
" output_dir='.',\n",
" group_by_length=True,\n",
" per_device_train_batch_size=8,\n",
" gradient_accumulation_steps=2,\n",
" evaluation_strategy=\"steps\",\n",
" gradient_checkpointing=True,\n",
" fp16=True,\n",
" num_train_epochs=30,\n",
" save_steps=100,\n",
" eval_steps=100,\n",
" logging_steps=100,\n",
" learning_rate=3e-5,\n",
" warmup_steps=500,\n",
" save_total_limit=3,\n",
" push_to_hub=True,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 63,
"id": "2a405bb3",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/workspace/xls-r-300m-km/. is already a clone of https://huggingface.co/vitouphy/xls-r-300m-km. Make sure you pull the latest changes with `repo.git_pull()`.\n",
"Using amp half precision backend\n"
]
}
],
"source": [
"from transformers import Trainer\n",
"\n",
"trainer = Trainer(\n",
" model=model,\n",
" data_collator=data_collator,\n",
" args=training_args,\n",
" compute_metrics=compute_metrics,\n",
" train_dataset=common_voice_train,\n",
" eval_dataset=common_voice_test,\n",
" tokenizer=processor.feature_extractor,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e9a99c77",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"The following columns in the training set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n",
"/opt/conda/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use thePyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
" warnings.warn(\n",
"***** Running training *****\n",
" Num examples = 2615\n",
" Num Epochs = 30\n",
" Instantaneous batch size per device = 8\n",
" Total train batch size (w. parallel, distributed & accumulation) = 16\n",
" Gradient Accumulation steps = 2\n",
" Total optimization steps = 4890\n"
]
},
{
"data": {
"text/html": [
"\n",
" \n",
" \n",
"
\n",
" [ 101/4890 01:40 < 1:21:11, 0.98 it/s, Epoch 0.61/30]\n",
"
\n",
" \n",
" \n",
" | Step | \n",
" Training Loss | \n",
" Validation Loss | \n",
" Cer | \n",
"
\n",
" \n",
" \n",
" \n",
" | 100 | \n",
" 16.976400 | \n",
" 13.300326 | \n",
" 0.989265 | \n",
"
\n",
" \n",
"
"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n",
"***** Running Evaluation *****\n",
" Num examples = 291\n",
" Batch size = 8\n",
"Saving model checkpoint to ./checkpoint-100\n",
"Configuration saved in ./checkpoint-100/config.json\n"
]
}
],
"source": [
"trainer.train()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3629e75f",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}