add training and visualization scripts and logs

Training/env.log

@@ -0,0 +1,107 @@
+SpeechBrain system description
+==============================
+Python version:
+3.9.7 (default, Sep 16 2021, 13:09:58) 
+[GCC 7.5.0]
+==============================
+Installed Python packages:
+aiohttp==3.8.1
+aiosignal==1.2.0
+appdirs==1.4.4
+async-timeout==4.0.2
+attrs==21.4.0
+audioread==2.1.9
+audiosegment==0.23.0
+azure-core==1.21.1
+azure-storage-blob==12.9.0
+bcrypt==3.2.0
+black==19.10b0
+certifi==2021.10.8
+cffi==1.15.0
+cfgv==3.3.1
+charset-normalizer==2.0.10
+click==8.0.3
+cryptography==36.0.1
+datasets==1.13.3
+decorator==5.1.1
+dill==0.3.4
+distlib==0.3.4
+entrypoints==0.3
+ffmpeg==1.4
+filelock==3.4.2
+flake8==3.7.9
+frozenlist==1.3.0
+fsspec==2022.2.0
+huggingface-hub==0.5.1
+HyperPyYAML==1.0.1
+identify==2.4.4
+idna==3.3
+isodate==0.6.1
+joblib==1.1.0
+librosa==0.8.1
+llvmlite==0.38.0
+mccabe==0.6.1
+more-itertools==8.12.0
+msrest==0.6.21
+multidict==6.0.2
+multiprocess==0.70.12.2
+mutagen==1.45.1
+nodeenv==1.6.0
+numba==0.55.0
+numpy==1.21.5
+oauthlib==3.1.1
+packaging==21.3
+pandas==1.3.5
+paramiko==2.10.3
+pathspec==0.9.0
+platformdirs==2.4.1
+pluggy==0.13.1
+pooch==1.5.2
+pre-commit==2.17.0
+py==1.11.0
+pyarrow==7.0.0
+pycodestyle==2.5.0
+pycparser==2.21
+pydub==0.25.1
+pyflakes==2.1.1
+PyNaCl==1.5.0
+pyparsing==3.0.6
+pytest==5.4.1
+python-dateutil==2.8.2
+pytz==2021.3
+PyYAML==6.0
+regex==2022.1.18
+requests==2.27.1
+requests-oauthlib==1.3.0
+resampy==0.2.2
+ruamel.yaml==0.17.21
+ruamel.yaml.clib==0.2.6
+sacremoses==0.0.53
+scikit-learn==1.0.2
+scipy==1.7.3
+scp==0.14.4
+sentencepiece==0.1.96
+six==1.16.0
+SoundFile==0.10.3.post1
+speechbrain==0.5.11
+threadpoolctl==3.0.0
+tokenizers==0.12.1
+toml==0.10.2
+torch==1.11.0
+torchaudio==0.11.0
+tqdm==4.62.3
+transformers==4.18.0
+typed-ast==1.5.1
+typing_extensions==4.0.1
+urllib3==1.26.8
+virtualenv==20.13.0
+wcwidth==0.2.5
+webrtcvad==2.0.10
+xxhash==3.0.0
+yamllint==1.23.0
+yarl==1.7.2
+youtube-dl==2021.12.17
+==============================
+Could not get git revision==============================
+Cuda version:
+10.2

Training/hyperparams.yaml

@@ -0,0 +1,193 @@
+# Generated 2022-05-27 from:
+# /data/n.abdoumohamed/dvoice-africa/speechbrain/recipes/DVoice/ASR/CTC/hparams/train_amharic.yaml
+# yamllint disable
+# ################################
+# Model: wav2vec2 + DNN + CTC
+# Augmentation: SpecAugment
+# Authors: Titouan Parcollet 2021
+# ################################
+
+# Seed needs to be set at top of yaml, before objects with parameters are made
+seed: 1249
+__set_seed: !!python/object/apply:torch.manual_see
+output_folder: results/wav2vec2_ctc_AMHARIC/1249
+wer_file: results/wav2vec2_ctc_AMHARIC/1249/wer.txt
+save_folder: results/wav2vec2_ctc_AMHARIC/1249/save
+train_log: results/wav2vec2_ctc_AMHARIC/1249/train_log.txt
+
+# URL for the biggest LeBenchmark wav2vec french.
+wav2vec2_hub: facebook/wav2vec2-large-xlsr-53
+
+# Data files
+data_folder: ASR/AMHARIC/data # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr
+train_csv_file: ASR/AMHARIC/data/train.csv # Standard CommonVoice .tsv files
+dev_csv_file: ASR/AMHARIC/data/dev.csv # Standard CommonVoice .tsv files
+test_csv_file: ASR/AMHARIC/data/test.csv # Standard CommonVoice .tsv files
+accented_letters: true
+language: amharic
+train_csv: results/wav2vec2_ctc_AMHARIC/save/train.csv
+valid_csv: results/wav2vec2_ctc_AMHARIC/save/dev.csv
+test_csv: results/wav2vec2_ctc_AMHARIC/save/test.csv
+skip_prep: false # Skip data preparation
+data_augmentation: false # Skip data augmentation
+
+# We remove utterance slonger than 10s in the train/dev/test sets as
+# longer sentences certainly correspond to "open microphones".
+avoid_if_longer_than: 15.0
+
+# Training parameters
+number_of_epochs: 30
+number_of_ctc_epochs: 15
+lr: 1.0
+lr_wav2vec: 0.0001
+ctc_weight: 0.3
+sorting: ascending
+auto_mix_prec: false
+sample_rate: 16000
+ckpt_interval_minutes: 30 # save checkpoint every N min
+
+# With data_parallel batch_size is split into N jobs
+# With DDP batch_size is multiplied by N jobs
+# Must be 6 per GPU to fit 16GB of VRAM
+batch_size: 4
+test_batch_size: 4
+
+dataloader_options:
+  batch_size: 4
+  num_workers: 2
+test_dataloader_options:
+  batch_size: 4
+  num_workers: 2
+
+# BPE parameters
+token_type: char  # ["unigram", "bpe", "char"]
+character_coverage: 1.0
+
+# Model parameters
+activation: !name:torch.nn.LeakyReLU
+wav2vec_output_dim: 1024
+dnn_neurons: 1024
+freeze_wav2vec: false
+
+# Outputs
+output_neurons: 224  # BPE size, index(blank/eos/bos) = 0
+
+# Decoding parameters
+# Be sure that the bos and eos index match with the BPEs ones
+blank_index: 0
+bos_index: 1
+eos_index: 2
+min_decode_ratio: 0.0
+max_decode_ratio: 1.0
+beam_size: 80
+eos_threshold: 1.5
+using_max_attn_shift: true
+max_attn_shift: 140
+ctc_weight_decode: 0.0
+temperature: 1.50
+
+#
+# Functions and classes
+#
+epoch_counter: &id007 !new:speechbrain.utils.epoch_loop.EpochCounter
+
+  limit: 30
+
+augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment
+  sample_rate: 16000
+  speeds: [95, 100, 105]
+
+enc: &id002 !new:speechbrain.nnet.containers.Sequential
+  input_shape: [null, null, 1024]
+  linear1: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn1: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation: !new:torch.nn.LeakyReLU
+  drop: !new:torch.nn.Dropout
+    p: 0.15
+  linear2: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn2: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation2: !new:torch.nn.LeakyReLU
+  drop2: !new:torch.nn.Dropout
+    p: 0.15
+  linear3: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn3: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation3: !new:torch.nn.LeakyReLU
+
+wav2vec2: &id001 !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2
+  source: facebook/wav2vec2-large-xlsr-53
+  output_norm: true
+  freeze: false
+  save_path: results/wav2vec2_ctc_AMHARIC/1249/save/wav2vec2_checkpoint
+
+#####
+# Uncomment this block if you prefer to use a Fairseq pretrained model instead
+# of a HuggingFace one. Here, we provide an URL that is obtained from the
+# Fairseq github for the multilingual XLSR.
+#
+#wav2vec2_url: https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr_53_56k.pt
+#wav2vec2: !new:speechbrain.lobes.models.fairseq_wav2vec.FairseqWav2Vec2
+#    pretrained_path: !ref <wav2vec2_url>
+#    output_norm: True
+#    freeze: False
+#    save_path: !ref <save_folder>/wav2vec2_checkpoint/model.pt
+#####
+
+
+ctc_lin: &id003 !new:speechbrain.nnet.linear.Linear
+
+  input_size: 1024
+  n_neurons: 224
+
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+  apply_log: true
+
+ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
+  blank_index: 0
+
+modules:
+  wav2vec2: *id001
+  enc: *id002
+  ctc_lin: *id003
+model: &id004 !new:torch.nn.ModuleList
+- [*id002, *id003]
+model_opt_class: !name:torch.optim.Adadelta
+  lr: 1.0
+  rho: 0.95
+  eps: 1.e-8
+
+wav2vec_opt_class: !name:torch.optim.Adam
+  lr: 0.0001
+
+lr_annealing_model: &id005 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 1.0
+  improvement_threshold: 0.0025
+  annealing_factor: 0.8
+  patient: 0
+
+lr_annealing_wav2vec: &id006 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 0.0001
+  improvement_threshold: 0.0025
+  annealing_factor: 0.9
+  patient: 0
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+  checkpoints_dir: results/wav2vec2_ctc_AMHARIC/1249/save
+  recoverables:
+    wav2vec2: *id001
+    model: *id004
+    scheduler_model: *id005
+    scheduler_wav2vec: *id006
+    counter: *id007
+train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
+  save_file: results/wav2vec2_ctc_AMHARIC/1249/train_log.txt
+
+error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+  split_tokens: true

Training/log.txt

@@ -0,0 +1,937 @@
+2022-04-05 14:25:26,074 - speechbrain.core - INFO - Beginning experiment!
+2022-04-05 14:25:26,075 - speechbrain.core - INFO - Experiment folder: results/wav2vec2_ctc_AMHARIC/1249
+2022-04-05 14:25:26,762 - speechbrain.utils.superpowers - DEBUG - aiohttp==3.8.1
+aiosignal==1.2.0
+appdirs==1.4.4
+async-timeout==4.0.2
+attrs==21.4.0
+audioread==2.1.9
+azure-core==1.21.1
+azure-storage-blob==12.9.0
+bcrypt==3.2.0
+black==19.10b0
+certifi==2021.10.8
+cffi==1.15.0
+cfgv==3.3.1
+charset-normalizer==2.0.10
+click==8.0.3
+cryptography==36.0.1
+datasets==1.13.3
+decorator==5.1.1
+dill==0.3.4
+distlib==0.3.4
+entrypoints==0.3
+ffmpeg==1.4
+filelock==3.4.2
+flake8==3.7.9
+frozenlist==1.3.0
+fsspec==2022.2.0
+huggingface-hub==0.2.1
+HyperPyYAML==1.0.0
+identify==2.4.4
+idna==3.3
+isodate==0.6.1
+joblib==1.1.0
+librosa==0.8.1
+llvmlite==0.38.0
+mccabe==0.6.1
+more-itertools==8.12.0
+msrest==0.6.21
+multidict==6.0.2
+multiprocess==0.70.12.2
+mutagen==1.45.1
+nodeenv==1.6.0
+numba==0.55.0
+numpy==1.21.5
+oauthlib==3.1.1
+packaging==21.3
+pandas==1.3.5
+paramiko==2.10.3
+pathspec==0.9.0
+platformdirs==2.4.1
+pluggy==0.13.1
+pooch==1.5.2
+pre-commit==2.17.0
+py==1.11.0
+pyarrow==7.0.0
+pycodestyle==2.5.0
+pycparser==2.21
+pyflakes==2.1.1
+PyNaCl==1.5.0
+pyparsing==3.0.6
+pytest==5.4.1
+python-dateutil==2.8.2
+pytz==2021.3
+PyYAML==6.0
+regex==2022.1.18
+requests==2.27.1
+requests-oauthlib==1.3.0
+resampy==0.2.2
+ruamel.yaml==0.17.20
+ruamel.yaml.clib==0.2.6
+sacremoses==0.0.47
+scikit-learn==1.0.2
+scipy==1.7.3
+scp==0.14.4
+sentencepiece==0.1.96
+six==1.16.0
+SoundFile==0.10.3.post1
+threadpoolctl==3.0.0
+tokenizers==0.10.3
+toml==0.10.2
+torch==1.10.1
+torchaudio==0.10.1
+tqdm==4.62.3
+transformers==4.13.0
+typed-ast==1.5.1
+typing_extensions==4.0.1
+urllib3==1.26.8
+virtualenv==20.13.0
+wcwidth==0.2.5
+xxhash==3.0.0
+yamllint==1.23.0
+yarl==1.7.2
+
+
+2022-04-05 14:25:26,922 - dvoice_prepare - INFO - Preparing CSV files for 7612 samples ...
+2022-04-05 14:25:26,922 - dvoice_prepare - INFO - Creating csv lists in results/wav2vec2_ctc_AMHARIC/1249/save/train.csv ...
+2022-04-05 14:25:27,322 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/train.csv successfully created!
+2022-04-05 14:25:27,322 - dvoice_prepare - INFO - Number of samples: 7612 
+2022-04-05 14:25:27,322 - dvoice_prepare - INFO - Total duration: 14.05 Hours
+2022-04-05 14:25:27,382 - dvoice_prepare - INFO - Preparing CSV files for 1631 samples ...
+2022-04-05 14:25:27,382 - dvoice_prepare - INFO - Creating csv lists in results/wav2vec2_ctc_AMHARIC/1249/save/dev.csv ...
+2022-04-05 14:25:27,434 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/dev.csv successfully created!
+2022-04-05 14:25:27,435 - dvoice_prepare - INFO - Number of samples: 1631 
+2022-04-05 14:25:27,435 - dvoice_prepare - INFO - Total duration: 2.95 Hours
+2022-04-05 14:25:27,477 - dvoice_prepare - INFO - Preparing CSV files for 1632 samples ...
+2022-04-05 14:25:27,477 - dvoice_prepare - INFO - Creating csv lists in results/wav2vec2_ctc_AMHARIC/1249/save/test.csv ...
+2022-04-05 14:25:27,546 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/test.csv successfully created!
+2022-04-05 14:25:27,546 - dvoice_prepare - INFO - Number of samples: 1632 
+2022-04-05 14:25:27,546 - dvoice_prepare - INFO - Total duration: 3.03 Hours
+2022-04-05 14:25:27,552 - speechbrain.tokenizers.SentencePiece - INFO - Train tokenizer with type:char
+2022-04-05 14:25:27,557 - speechbrain.tokenizers.SentencePiece - INFO - Extract wrd sequences from:results/wav2vec2_ctc_AMHARIC/1249/save/train.csv
+2022-04-05 14:25:27,707 - speechbrain.tokenizers.SentencePiece - INFO - Text file created at: results/wav2vec2_ctc_AMHARIC/1249/save/train.txt
+2022-04-05 14:25:27,870 - speechbrain.tokenizers.SentencePiece - INFO - ==== Loading Tokenizer ===
+2022-04-05 14:25:27,870 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer path: results/wav2vec2_ctc_AMHARIC/1249/save/224_char.model
+2022-04-05 14:25:27,870 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer vocab_size: 224
+2022-04-05 14:25:27,870 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer type: char
+2022-04-05 14:25:27,990 - speechbrain.core - INFO - Info: auto_mix_prec arg from hparam file is used
+2022-04-05 14:25:27,991 - speechbrain.core - INFO - Info: ckpt_interval_minutes arg from hparam file is used
+2022-04-05 14:25:30,936 - speechbrain.core - INFO - 318.8M trainable parameters in ASR
+2022-04-05 14:25:30,940 - speechbrain.utils.checkpoints - INFO - Would load a checkpoint here, but none found yet.
+2022-04-05 14:25:30,940 - speechbrain.utils.epoch_loop - INFO - Going into epoch 1
+2022-04-05 14:37:08,995 - speechbrain.core - ERROR - Exception:
+Traceback (most recent call last):
+  File "/data/n.abdoumohamed/DVoice/speechbrain/recipes/DVoice/ASR/CTC/train2.py", line 366, in <module>
+    asr_brain.fit(
+  File "/data/n.abdoumohamed/DVoice/speechbrain/speechbrain/core.py", line 1034, in fit
+    loss = self.fit_batch(batch)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/recipes/DVoice/ASR/CTC/train2.py", line 120, in fit_batch
+    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/recipes/DVoice/ASR/CTC/train2.py", line 61, in compute_forward
+    feats = self.modules.wav2vec2(wavs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/speechbrain/lobes/models/huggingface_wav2vec.py", line 254, in forward
+    return self.extract_features(wav)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/speechbrain/lobes/models/huggingface_wav2vec.py", line 269, in extract_features
+    out = self.model(wav)[0]
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 1253, in forward
+    encoder_outputs = self.encoder(
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 846, in forward
+    layer_outputs = layer(
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 687, in forward
+    hidden_states = hidden_states + self.feed_forward(self.final_layer_norm(hidden_states))
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 622, in forward
+    hidden_states = self.intermediate_dense(hidden_states)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/linear.py", line 103, in forward
+    return F.linear(input, self.weight, self.bias)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/functional.py", line 1848, in linear
+    return torch._C._nn.linear(input, weight, bias)
+RuntimeError: CUDA out of memory. Tried to allocate 24.00 MiB (GPU 0; 11.91 GiB total capacity; 10.77 GiB already allocated; 25.25 MiB free; 10.94 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
+2022-04-05 21:06:45,197 - speechbrain.core - INFO - Beginning experiment!
+2022-04-05 21:06:45,198 - speechbrain.core - INFO - Experiment folder: results/wav2vec2_ctc_AMHARIC/1249
+2022-04-05 21:06:45,836 - speechbrain.utils.superpowers - DEBUG - aiohttp==3.8.1
+aiosignal==1.2.0
+appdirs==1.4.4
+async-timeout==4.0.2
+attrs==21.4.0
+audioread==2.1.9
+azure-core==1.21.1
+azure-storage-blob==12.9.0
+bcrypt==3.2.0
+black==19.10b0
+certifi==2021.10.8
+cffi==1.15.0
+cfgv==3.3.1
+charset-normalizer==2.0.10
+click==8.0.3
+cryptography==36.0.1
+datasets==1.13.3
+decorator==5.1.1
+dill==0.3.4
+distlib==0.3.4
+entrypoints==0.3
+ffmpeg==1.4
+filelock==3.4.2
+flake8==3.7.9
+frozenlist==1.3.0
+fsspec==2022.2.0
+huggingface-hub==0.2.1
+HyperPyYAML==1.0.0
+identify==2.4.4
+idna==3.3
+isodate==0.6.1
+joblib==1.1.0
+librosa==0.8.1
+llvmlite==0.38.0
+mccabe==0.6.1
+more-itertools==8.12.0
+msrest==0.6.21
+multidict==6.0.2
+multiprocess==0.70.12.2
+mutagen==1.45.1
+nodeenv==1.6.0
+numba==0.55.0
+numpy==1.21.5
+oauthlib==3.1.1
+packaging==21.3
+pandas==1.3.5
+paramiko==2.10.3
+pathspec==0.9.0
+platformdirs==2.4.1
+pluggy==0.13.1
+pooch==1.5.2
+pre-commit==2.17.0
+py==1.11.0
+pyarrow==7.0.0
+pycodestyle==2.5.0
+pycparser==2.21
+pyflakes==2.1.1
+PyNaCl==1.5.0
+pyparsing==3.0.6
+pytest==5.4.1
+python-dateutil==2.8.2
+pytz==2021.3
+PyYAML==6.0
+regex==2022.1.18
+requests==2.27.1
+requests-oauthlib==1.3.0
+resampy==0.2.2
+ruamel.yaml==0.17.20
+ruamel.yaml.clib==0.2.6
+sacremoses==0.0.47
+scikit-learn==1.0.2
+scipy==1.7.3
+scp==0.14.4
+sentencepiece==0.1.96
+six==1.16.0
+SoundFile==0.10.3.post1
+threadpoolctl==3.0.0
+tokenizers==0.10.3
+toml==0.10.2
+torch==1.10.1
+torchaudio==0.10.1
+tqdm==4.62.3
+transformers==4.13.0
+typed-ast==1.5.1
+typing_extensions==4.0.1
+urllib3==1.26.8
+virtualenv==20.13.0
+wcwidth==0.2.5
+xxhash==3.0.0
+yamllint==1.23.0
+yarl==1.7.2
+
+
+2022-04-05 21:06:46,007 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/train.csv already exists, skipping data preparation!
+2022-04-05 21:06:46,007 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/dev.csv already exists, skipping data preparation!
+2022-04-05 21:06:46,007 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/test.csv already exists, skipping data preparation!
+2022-04-05 21:06:46,007 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer is already trained.
+2022-04-05 21:06:46,007 - speechbrain.tokenizers.SentencePiece - INFO - ==== Loading Tokenizer ===
+2022-04-05 21:06:46,007 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer path: results/wav2vec2_ctc_AMHARIC/1249/save/224_char.model
+2022-04-05 21:06:46,007 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer vocab_size: 224
+2022-04-05 21:06:46,007 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer type: char
+2022-04-05 21:06:46,145 - speechbrain.core - INFO - Info: auto_mix_prec arg from hparam file is used
+2022-04-05 21:06:46,145 - speechbrain.core - INFO - Info: ckpt_interval_minutes arg from hparam file is used
+2022-04-05 21:06:49,137 - speechbrain.core - INFO - 318.8M trainable parameters in ASR
+2022-04-05 21:06:49,142 - speechbrain.utils.checkpoints - INFO - Would load a checkpoint here, but none found yet.
+2022-04-05 21:06:49,142 - speechbrain.utils.epoch_loop - INFO - Going into epoch 1
+2022-04-05 21:18:09,453 - speechbrain.core - ERROR - Exception:
+Traceback (most recent call last):
+  File "/data/n.abdoumohamed/DVoice/speechbrain/recipes/DVoice/ASR/CTC/train2.py", line 366, in <module>
+    asr_brain.fit(
+  File "/data/n.abdoumohamed/DVoice/speechbrain/speechbrain/core.py", line 1034, in fit
+    loss = self.fit_batch(batch)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/recipes/DVoice/ASR/CTC/train2.py", line 120, in fit_batch
+    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/recipes/DVoice/ASR/CTC/train2.py", line 61, in compute_forward
+    feats = self.modules.wav2vec2(wavs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/speechbrain/lobes/models/huggingface_wav2vec.py", line 254, in forward
+    return self.extract_features(wav)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/speechbrain/lobes/models/huggingface_wav2vec.py", line 269, in extract_features
+    out = self.model(wav)[0]
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 1253, in forward
+    encoder_outputs = self.encoder(
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 846, in forward
+    layer_outputs = layer(
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 687, in forward
+    hidden_states = hidden_states + self.feed_forward(self.final_layer_norm(hidden_states))
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 626, in forward
+    hidden_states = self.output_dense(hidden_states)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/linear.py", line 103, in forward
+    return F.linear(input, self.weight, self.bias)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/functional.py", line 1848, in linear
+    return torch._C._nn.linear(input, weight, bias)
+RuntimeError: CUDA out of memory. Tried to allocate 20.00 MiB (GPU 0; 11.91 GiB total capacity; 10.65 GiB already allocated; 17.25 MiB free; 10.95 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
+2022-04-06 10:21:42,642 - speechbrain.core - INFO - Beginning experiment!
+2022-04-06 10:21:42,643 - speechbrain.core - INFO - Experiment folder: results/wav2vec2_ctc_AMHARIC/1249
+2022-04-06 10:21:43,318 - speechbrain.utils.superpowers - DEBUG - aiohttp==3.8.1
+aiosignal==1.2.0
+appdirs==1.4.4
+async-timeout==4.0.2
+attrs==21.4.0
+audioread==2.1.9
+azure-core==1.21.1
+azure-storage-blob==12.9.0
+bcrypt==3.2.0
+black==19.10b0
+certifi==2021.10.8
+cffi==1.15.0
+cfgv==3.3.1
+charset-normalizer==2.0.10
+click==8.0.3
+cryptography==36.0.1
+datasets==1.13.3
+decorator==5.1.1
+dill==0.3.4
+distlib==0.3.4
+entrypoints==0.3
+ffmpeg==1.4
+filelock==3.4.2
+flake8==3.7.9
+frozenlist==1.3.0
+fsspec==2022.2.0
+huggingface-hub==0.2.1
+HyperPyYAML==1.0.0
+identify==2.4.4
+idna==3.3
+isodate==0.6.1
+joblib==1.1.0
+librosa==0.8.1
+llvmlite==0.38.0
+mccabe==0.6.1
+more-itertools==8.12.0
+msrest==0.6.21
+multidict==6.0.2
+multiprocess==0.70.12.2
+mutagen==1.45.1
+nodeenv==1.6.0
+numba==0.55.0
+numpy==1.21.5
+oauthlib==3.1.1
+packaging==21.3
+pandas==1.3.5
+paramiko==2.10.3
+pathspec==0.9.0
+platformdirs==2.4.1
+pluggy==0.13.1
+pooch==1.5.2
+pre-commit==2.17.0
+py==1.11.0
+pyarrow==7.0.0
+pycodestyle==2.5.0
+pycparser==2.21
+pyflakes==2.1.1
+PyNaCl==1.5.0
+pyparsing==3.0.6
+pytest==5.4.1
+python-dateutil==2.8.2
+pytz==2021.3
+PyYAML==6.0
+regex==2022.1.18
+requests==2.27.1
+requests-oauthlib==1.3.0
+resampy==0.2.2
+ruamel.yaml==0.17.20
+ruamel.yaml.clib==0.2.6
+sacremoses==0.0.47
+scikit-learn==1.0.2
+scipy==1.7.3
+scp==0.14.4
+sentencepiece==0.1.96
+six==1.16.0
+SoundFile==0.10.3.post1
+threadpoolctl==3.0.0
+tokenizers==0.10.3
+toml==0.10.2
+torch==1.10.1
+torchaudio==0.10.1
+tqdm==4.62.3
+transformers==4.13.0
+typed-ast==1.5.1
+typing_extensions==4.0.1
+urllib3==1.26.8
+virtualenv==20.13.0
+wcwidth==0.2.5
+xxhash==3.0.0
+yamllint==1.23.0
+yarl==1.7.2
+
+
+2022-04-06 10:21:43,439 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/train.csv already exists, skipping data preparation!
+2022-04-06 10:21:43,439 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/dev.csv already exists, skipping data preparation!
+2022-04-06 10:21:43,439 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/test.csv already exists, skipping data preparation!
+2022-04-06 10:21:43,439 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer is already trained.
+2022-04-06 10:21:43,439 - speechbrain.tokenizers.SentencePiece - INFO - ==== Loading Tokenizer ===
+2022-04-06 10:21:43,439 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer path: results/wav2vec2_ctc_AMHARIC/1249/save/224_char.model
+2022-04-06 10:21:43,440 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer vocab_size: 224
+2022-04-06 10:21:43,440 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer type: char
+2022-04-06 10:21:43,580 - speechbrain.core - INFO - Info: auto_mix_prec arg from hparam file is used
+2022-04-06 10:21:43,580 - speechbrain.core - INFO - Info: ckpt_interval_minutes arg from hparam file is used
+2022-04-06 10:21:46,557 - speechbrain.core - INFO - 318.8M trainable parameters in ASR
+2022-04-06 10:21:46,562 - speechbrain.utils.checkpoints - INFO - Would load a checkpoint here, but none found yet.
+2022-04-06 10:21:46,562 - speechbrain.utils.epoch_loop - INFO - Going into epoch 1
+2022-04-06 10:45:30,095 - speechbrain.core - ERROR - Exception:
+Traceback (most recent call last):
+  File "/data/n.abdoumohamed/DVoice/speechbrain/recipes/DVoice/ASR/CTC/train2.py", line 366, in <module>
+    asr_brain.fit(
+  File "/data/n.abdoumohamed/DVoice/speechbrain/speechbrain/core.py", line 1034, in fit
+    loss = self.fit_batch(batch)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/recipes/DVoice/ASR/CTC/train2.py", line 120, in fit_batch
+    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/recipes/DVoice/ASR/CTC/train2.py", line 61, in compute_forward
+    feats = self.modules.wav2vec2(wavs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/speechbrain/lobes/models/huggingface_wav2vec.py", line 254, in forward
+    return self.extract_features(wav)
+  File "/data/n.abdoumohamed/DVoice/speechbrain/speechbrain/lobes/models/huggingface_wav2vec.py", line 269, in extract_features
+    out = self.model(wav)[0]
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 1253, in forward
+    encoder_outputs = self.encoder(
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 846, in forward
+    layer_outputs = layer(
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 682, in forward
+    hidden_states, attn_weights, _ = self.attention(
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
+    return forward_call(*input, **kwargs)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 586, in forward
+    attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)
+  File "/home/n.abdoumohamed/.conda/envs/speechbrain/lib/python3.9/site-packages/torch/nn/functional.py", line 1169, in dropout
+    return _VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training)
+RuntimeError: CUDA out of memory. Tried to allocate 46.00 MiB (GPU 0; 11.91 GiB total capacity; 10.61 GiB already allocated; 5.25 MiB free; 10.96 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
+2022-04-07 01:24:51,494 - speechbrain.core - INFO - Beginning experiment!
+2022-04-07 01:24:51,498 - speechbrain.core - INFO - Experiment folder: results/wav2vec2_ctc_AMHARIC/1249
+2022-04-07 01:24:52,086 - speechbrain.utils.superpowers - DEBUG - aiohttp==3.8.1
+aiosignal==1.2.0
+appdirs==1.4.4
+async-timeout==4.0.2
+attrs==21.4.0
+audioread==2.1.9
+azure-core==1.21.1
+azure-storage-blob==12.9.0
+bcrypt==3.2.0
+black==19.10b0
+certifi==2021.10.8
+cffi==1.15.0
+cfgv==3.3.1
+charset-normalizer==2.0.10
+click==8.0.3
+cryptography==36.0.1
+datasets==1.13.3
+decorator==5.1.1
+dill==0.3.4
+distlib==0.3.4
+entrypoints==0.3
+ffmpeg==1.4
+filelock==3.4.2
+flake8==3.7.9
+frozenlist==1.3.0
+fsspec==2022.2.0
+huggingface-hub==0.2.1
+HyperPyYAML==1.0.0
+identify==2.4.4
+idna==3.3
+isodate==0.6.1
+joblib==1.1.0
+librosa==0.8.1
+llvmlite==0.38.0
+mccabe==0.6.1
+more-itertools==8.12.0
+msrest==0.6.21
+multidict==6.0.2
+multiprocess==0.70.12.2
+mutagen==1.45.1
+nodeenv==1.6.0
+numba==0.55.0
+numpy==1.21.5
+oauthlib==3.1.1
+packaging==21.3
+pandas==1.3.5
+paramiko==2.10.3
+pathspec==0.9.0
+platformdirs==2.4.1
+pluggy==0.13.1
+pooch==1.5.2
+pre-commit==2.17.0
+py==1.11.0
+pyarrow==7.0.0
+pycodestyle==2.5.0
+pycparser==2.21
+pyflakes==2.1.1
+PyNaCl==1.5.0
+pyparsing==3.0.6
+pytest==5.4.1
+python-dateutil==2.8.2
+pytz==2021.3
+PyYAML==6.0
+regex==2022.1.18
+requests==2.27.1
+requests-oauthlib==1.3.0
+resampy==0.2.2
+ruamel.yaml==0.17.20
+ruamel.yaml.clib==0.2.6
+sacremoses==0.0.47
+scikit-learn==1.0.2
+scipy==1.7.3
+scp==0.14.4
+sentencepiece==0.1.96
+six==1.16.0
+SoundFile==0.10.3.post1
+threadpoolctl==3.0.0
+tokenizers==0.10.3
+toml==0.10.2
+torch==1.10.1
+torchaudio==0.10.1
+tqdm==4.62.3
+transformers==4.13.0
+typed-ast==1.5.1
+typing_extensions==4.0.1
+urllib3==1.26.8
+virtualenv==20.13.0
+wcwidth==0.2.5
+xxhash==3.0.0
+yamllint==1.23.0
+yarl==1.7.2
+
+
+2022-04-07 01:24:52,319 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/train.csv already exists, skipping data preparation!
+2022-04-07 01:24:52,319 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/dev.csv already exists, skipping data preparation!
+2022-04-07 01:24:52,319 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/test.csv already exists, skipping data preparation!
+2022-04-07 01:24:52,323 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer is already trained.
+2022-04-07 01:24:52,323 - speechbrain.tokenizers.SentencePiece - INFO - ==== Loading Tokenizer ===
+2022-04-07 01:24:52,323 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer path: results/wav2vec2_ctc_AMHARIC/1249/save/224_char.model
+2022-04-07 01:24:52,323 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer vocab_size: 224
+2022-04-07 01:24:52,323 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer type: char
+2022-04-07 01:24:52,988 - speechbrain.core - INFO - Info: auto_mix_prec arg from hparam file is used
+2022-04-07 01:24:52,988 - speechbrain.core - INFO - Info: ckpt_interval_minutes arg from hparam file is used
+2022-04-07 01:24:56,199 - speechbrain.core - INFO - 318.8M trainable parameters in ASR
+2022-04-07 01:24:56,215 - speechbrain.utils.checkpoints - INFO - Would load a checkpoint here, but none found yet.
+2022-04-07 01:24:56,215 - speechbrain.utils.epoch_loop - INFO - Going into epoch 1
+2022-04-07 01:39:30,997 - speechbrain.utils.train_logger - INFO - epoch: 1, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 2.16 - valid loss: 6.31e-01, valid CER: 17.94, valid WER: 61.08
+2022-04-07 01:41:12,908 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+01-39-31+00
+2022-04-07 01:41:13,081 - speechbrain.utils.epoch_loop - INFO - Going into epoch 2
+2022-04-07 01:54:26,673 - speechbrain.utils.train_logger - INFO - epoch: 2, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 7.60e-01 - valid loss: 4.76e-01, valid CER: 13.59, valid WER: 49.70
+2022-04-07 01:56:27,270 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+01-54-26+00
+2022-04-07 01:56:28,698 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+01-39-31+00
+2022-04-07 01:56:28,698 - speechbrain.utils.epoch_loop - INFO - Going into epoch 3
+2022-04-07 02:09:43,082 - speechbrain.utils.train_logger - INFO - epoch: 3, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 5.88e-01 - valid loss: 4.04e-01, valid CER: 11.77, valid WER: 43.35
+2022-04-07 02:14:21,215 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+02-09-43+00
+2022-04-07 02:14:22,071 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+01-54-26+00
+2022-04-07 02:14:22,071 - speechbrain.utils.epoch_loop - INFO - Going into epoch 4
+2022-04-07 02:27:35,021 - speechbrain.utils.train_logger - INFO - epoch: 4, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 4.97e-01 - valid loss: 3.75e-01, valid CER: 10.61, valid WER: 39.29
+2022-04-07 02:28:59,128 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+02-27-35+00
+2022-04-07 02:29:00,277 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+02-09-43+00
+2022-04-07 02:29:00,277 - speechbrain.utils.epoch_loop - INFO - Going into epoch 5
+2022-04-07 02:42:13,500 - speechbrain.utils.train_logger - INFO - epoch: 5, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 4.29e-01 - valid loss: 3.71e-01, valid CER: 10.15, valid WER: 37.38
+2022-04-07 02:47:20,435 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+02-42-13+00
+2022-04-07 02:47:21,615 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+02-27-35+00
+2022-04-07 02:47:21,615 - speechbrain.utils.epoch_loop - INFO - Going into epoch 6
+2022-04-07 03:00:32,602 - speechbrain.utils.train_logger - INFO - epoch: 6, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 3.81e-01 - valid loss: 3.54e-01, valid CER: 9.54, valid WER: 35.23
+2022-04-07 03:02:35,290 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+03-00-32+00
+2022-04-07 03:02:35,924 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+02-42-13+00
+2022-04-07 03:02:35,925 - speechbrain.utils.epoch_loop - INFO - Going into epoch 7
+2022-04-07 03:15:54,749 - speechbrain.utils.train_logger - INFO - epoch: 7, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 3.39e-01 - valid loss: 3.41e-01, valid CER: 8.98, valid WER: 33.48
+2022-04-07 03:17:35,622 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+03-15-54+00
+2022-04-07 03:17:36,628 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+03-00-32+00
+2022-04-07 03:17:36,628 - speechbrain.utils.epoch_loop - INFO - Going into epoch 8
+2022-04-07 03:30:53,354 - speechbrain.nnet.schedulers - INFO - Changing lr from 1 to 0.8
+2022-04-07 03:30:53,376 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.0001 to 9e-05
+2022-04-07 03:30:53,520 - speechbrain.utils.train_logger - INFO - epoch: 8, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 3.08e-01 - valid loss: 3.57e-01, valid CER: 8.80, valid WER: 32.41
+2022-04-07 03:33:47,026 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+03-30-53+00
+2022-04-07 03:33:48,209 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+03-15-54+00
+2022-04-07 03:33:48,209 - speechbrain.utils.epoch_loop - INFO - Going into epoch 9
+2022-04-07 03:46:59,930 - speechbrain.utils.train_logger - INFO - epoch: 9, lr_model: 8.00e-01, lr_wav2vec: 9.00e-05 - train loss: 2.70e-01 - valid loss: 3.46e-01, valid CER: 8.47, valid WER: 31.33
+2022-04-07 03:48:26,079 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+03-46-59+00
+2022-04-07 03:48:26,901 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+03-30-53+00
+2022-04-07 03:48:26,901 - speechbrain.utils.epoch_loop - INFO - Going into epoch 10
+2022-04-07 04:01:43,511 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.8 to 0.64
+2022-04-07 04:01:43,527 - speechbrain.nnet.schedulers - INFO - Changing lr from 9e-05 to 8.1e-05
+2022-04-07 04:01:43,843 - speechbrain.utils.train_logger - INFO - epoch: 10, lr_model: 8.00e-01, lr_wav2vec: 9.00e-05 - train loss: 2.45e-01 - valid loss: 3.64e-01, valid CER: 8.30, valid WER: 30.31
+2022-04-07 04:03:02,059 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+04-01-43+00
+2022-04-07 04:03:02,966 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+03-46-59+00
+2022-04-07 04:03:02,966 - speechbrain.utils.epoch_loop - INFO - Going into epoch 11
+2022-04-07 04:16:16,977 - speechbrain.utils.train_logger - INFO - epoch: 11, lr_model: 6.40e-01, lr_wav2vec: 8.10e-05 - train loss: 2.17e-01 - valid loss: 3.43e-01, valid CER: 8.00, valid WER: 29.91
+2022-04-07 04:17:41,125 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+04-16-16+00
+2022-04-07 04:17:42,305 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+04-01-43+00
+2022-04-07 04:17:42,305 - speechbrain.utils.epoch_loop - INFO - Going into epoch 12
+2022-04-07 04:30:56,259 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.64 to 0.51
+2022-04-07 04:30:56,276 - speechbrain.nnet.schedulers - INFO - Changing lr from 8.1e-05 to 7.3e-05
+2022-04-07 04:30:56,372 - speechbrain.utils.train_logger - INFO - epoch: 12, lr_model: 6.40e-01, lr_wav2vec: 8.10e-05 - train loss: 1.98e-01 - valid loss: 3.68e-01, valid CER: 7.93, valid WER: 29.49
+2022-04-07 04:33:36,676 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+04-30-56+00
+2022-04-07 04:33:38,939 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+04-16-16+00
+2022-04-07 04:33:38,939 - speechbrain.utils.epoch_loop - INFO - Going into epoch 13
+2022-04-07 04:46:50,071 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.51 to 0.41
+2022-04-07 04:46:50,283 - speechbrain.nnet.schedulers - INFO - Changing lr from 7.3e-05 to 6.6e-05
+2022-04-07 04:46:50,494 - speechbrain.utils.train_logger - INFO - epoch: 13, lr_model: 5.12e-01, lr_wav2vec: 7.29e-05 - train loss: 1.75e-01 - valid loss: 3.94e-01, valid CER: 7.78, valid WER: 29.09
+2022-04-07 04:49:19,543 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+04-46-50+00
+2022-04-07 04:49:20,921 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+04-30-56+00
+2022-04-07 04:49:20,921 - speechbrain.utils.epoch_loop - INFO - Going into epoch 14
+2022-04-07 05:02:32,796 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.41 to 0.33
+2022-04-07 05:02:32,805 - speechbrain.nnet.schedulers - INFO - Changing lr from 6.6e-05 to 5.9e-05
+2022-04-07 05:02:32,852 - speechbrain.utils.train_logger - INFO - epoch: 14, lr_model: 4.10e-01, lr_wav2vec: 6.56e-05 - train loss: 1.58e-01 - valid loss: 3.94e-01, valid CER: 7.75, valid WER: 28.90
+2022-04-07 05:04:12,624 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+05-02-32+00
+2022-04-07 05:04:14,019 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+04-46-50+00
+2022-04-07 05:04:14,019 - speechbrain.utils.epoch_loop - INFO - Going into epoch 15
+2022-04-07 05:17:40,541 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.33 to 0.26
+2022-04-07 05:17:40,556 - speechbrain.nnet.schedulers - INFO - Changing lr from 5.9e-05 to 5.3e-05
+2022-04-07 05:17:40,606 - speechbrain.utils.train_logger - INFO - epoch: 15, lr_model: 3.28e-01, lr_wav2vec: 5.90e-05 - train loss: 1.37e-01 - valid loss: 4.13e-01, valid CER: 7.64, valid WER: 28.39
+2022-04-07 05:19:24,936 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+05-17-40+00
+2022-04-07 05:19:25,870 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+05-02-32+00
+2022-04-07 05:19:25,870 - speechbrain.utils.epoch_loop - INFO - Going into epoch 16
+2022-04-07 05:32:43,793 - speechbrain.utils.train_logger - INFO - epoch: 16, lr_model: 2.62e-01, lr_wav2vec: 5.31e-05 - train loss: 1.25e-01 - valid loss: 3.95e-01, valid CER: 7.48, valid WER: 27.94
+2022-04-07 05:34:31,869 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+05-32-43+00
+2022-04-07 05:34:32,562 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+05-17-40+00
+2022-04-07 05:34:32,563 - speechbrain.utils.epoch_loop - INFO - Going into epoch 17
+2022-04-07 05:47:42,780 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.26 to 0.21
+2022-04-07 05:47:42,796 - speechbrain.nnet.schedulers - INFO - Changing lr from 5.3e-05 to 4.8e-05
+2022-04-07 05:47:42,879 - speechbrain.utils.train_logger - INFO - epoch: 17, lr_model: 2.62e-01, lr_wav2vec: 5.31e-05 - train loss: 1.20e-01 - valid loss: 4.12e-01, valid CER: 7.36, valid WER: 27.73
+2022-04-07 05:50:08,133 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+05-47-42+00
+2022-04-07 05:50:11,233 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+05-32-43+00
+2022-04-07 05:50:11,244 - speechbrain.utils.epoch_loop - INFO - Going into epoch 18
+2022-04-07 06:03:30,233 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.21 to 0.17
+2022-04-07 06:03:30,246 - speechbrain.nnet.schedulers - INFO - Changing lr from 4.8e-05 to 4.3e-05
+2022-04-07 06:03:30,328 - speechbrain.utils.train_logger - INFO - epoch: 18, lr_model: 2.10e-01, lr_wav2vec: 4.78e-05 - train loss: 1.03e-01 - valid loss: 4.31e-01, valid CER: 7.44, valid WER: 27.69
+2022-04-07 06:07:59,476 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+06-03-30+00
+2022-04-07 06:08:02,491 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+05-47-42+00
+2022-04-07 06:08:02,503 - speechbrain.utils.epoch_loop - INFO - Going into epoch 19
+2022-04-07 06:21:15,279 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.17 to 0.13
+2022-04-07 06:21:15,296 - speechbrain.nnet.schedulers - INFO - Changing lr from 4.3e-05 to 3.9e-05
+2022-04-07 06:21:15,414 - speechbrain.utils.train_logger - INFO - epoch: 19, lr_model: 1.68e-01, lr_wav2vec: 4.30e-05 - train loss: 9.82e-02 - valid loss: 4.35e-01, valid CER: 7.28, valid WER: 27.08
+2022-04-07 06:23:34,904 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+06-21-15+00
+2022-04-07 06:23:36,249 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+06-03-30+00
+2022-04-07 06:23:36,250 - speechbrain.utils.epoch_loop - INFO - Going into epoch 20
+2022-04-07 06:36:49,810 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.13 to 0.11
+2022-04-07 06:36:49,823 - speechbrain.nnet.schedulers - INFO - Changing lr from 3.9e-05 to 3.5e-05
+2022-04-07 06:36:49,890 - speechbrain.utils.train_logger - INFO - epoch: 20, lr_model: 1.34e-01, lr_wav2vec: 3.87e-05 - train loss: 8.78e-02 - valid loss: 4.42e-01, valid CER: 7.27, valid WER: 27.18
+2022-04-07 06:39:22,101 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+06-36-49+00
+2022-04-07 06:39:22,945 - speechbrain.utils.epoch_loop - INFO - Going into epoch 21
+2022-04-07 06:52:36,778 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.11 to 0.086
+2022-04-07 06:52:36,782 - speechbrain.nnet.schedulers - INFO - Changing lr from 3.5e-05 to 3.1e-05
+2022-04-07 06:52:36,852 - speechbrain.utils.train_logger - INFO - epoch: 21, lr_model: 1.07e-01, lr_wav2vec: 3.49e-05 - train loss: 8.27e-02 - valid loss: 4.72e-01, valid CER: 7.16, valid WER: 26.79
+2022-04-07 06:55:10,567 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+06-52-36+00
+2022-04-07 06:55:12,223 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+06-21-15+00
+2022-04-07 06:55:12,905 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+06-36-49+00
+2022-04-07 06:55:12,905 - speechbrain.utils.epoch_loop - INFO - Going into epoch 22
+2022-04-07 07:08:24,492 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.086 to 0.069
+2022-04-07 07:08:24,497 - speechbrain.nnet.schedulers - INFO - Changing lr from 3.1e-05 to 2.8e-05
+2022-04-07 07:08:24,542 - speechbrain.utils.train_logger - INFO - epoch: 22, lr_model: 8.59e-02, lr_wav2vec: 3.14e-05 - train loss: 7.39e-02 - valid loss: 4.88e-01, valid CER: 7.03, valid WER: 26.28
+2022-04-07 07:10:26,930 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+07-08-24+00
+2022-04-07 07:10:29,254 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+06-52-36+00
+2022-04-07 07:10:29,254 - speechbrain.utils.epoch_loop - INFO - Going into epoch 23
+2022-04-07 07:23:43,676 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.069 to 0.055
+2022-04-07 07:23:43,692 - speechbrain.nnet.schedulers - INFO - Changing lr from 2.8e-05 to 2.5e-05
+2022-04-07 07:23:43,814 - speechbrain.utils.train_logger - INFO - epoch: 23, lr_model: 6.87e-02, lr_wav2vec: 2.82e-05 - train loss: 7.24e-02 - valid loss: 4.92e-01, valid CER: 6.95, valid WER: 26.03
+2022-04-07 07:29:42,385 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+07-23-43+00
+2022-04-07 07:29:44,066 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+07-08-24+00
+2022-04-07 07:29:44,066 - speechbrain.utils.epoch_loop - INFO - Going into epoch 24
+2022-04-07 07:43:02,440 - speechbrain.utils.train_logger - INFO - epoch: 24, lr_model: 5.50e-02, lr_wav2vec: 2.54e-05 - train loss: 6.76e-02 - valid loss: 4.80e-01, valid CER: 6.89, valid WER: 26.11
+2022-04-07 07:48:09,873 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+07-43-02+00
+2022-04-07 07:48:12,476 - speechbrain.utils.epoch_loop - INFO - Going into epoch 25
+2022-04-07 08:01:25,251 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.055 to 0.044
+2022-04-07 08:01:25,297 - speechbrain.nnet.schedulers - INFO - Changing lr from 2.5e-05 to 2.3e-05
+2022-04-07 08:01:25,429 - speechbrain.utils.train_logger - INFO - epoch: 25, lr_model: 5.50e-02, lr_wav2vec: 2.54e-05 - train loss: 6.21e-02 - valid loss: 5.01e-01, valid CER: 6.88, valid WER: 25.88
+2022-04-07 08:03:25,750 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+08-01-25+00
+2022-04-07 08:03:27,595 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+07-43-02+00
+2022-04-07 08:03:28,215 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+07-23-43+00
+2022-04-07 08:03:28,215 - speechbrain.utils.epoch_loop - INFO - Going into epoch 26
+2022-04-07 08:16:35,211 - speechbrain.utils.train_logger - INFO - epoch: 26, lr_model: 4.40e-02, lr_wav2vec: 2.29e-05 - train loss: 5.94e-02 - valid loss: 4.89e-01, valid CER: 6.90, valid WER: 26.08
+2022-04-07 08:19:37,126 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+08-16-35+00
+2022-04-07 08:19:39,166 - speechbrain.utils.epoch_loop - INFO - Going into epoch 27
+2022-04-07 08:32:51,788 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.044 to 0.035
+2022-04-07 08:32:51,994 - speechbrain.nnet.schedulers - INFO - Changing lr from 2.3e-05 to 2.1e-05
+2022-04-07 08:32:52,086 - speechbrain.utils.train_logger - INFO - epoch: 27, lr_model: 4.40e-02, lr_wav2vec: 2.29e-05 - train loss: 5.54e-02 - valid loss: 5.09e-01, valid CER: 6.87, valid WER: 26.04
+2022-04-07 08:35:48,324 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+08-32-52+00
+2022-04-07 08:35:51,566 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+08-16-35+00
+2022-04-07 08:35:51,582 - speechbrain.utils.epoch_loop - INFO - Going into epoch 28
+2022-04-07 08:48:53,259 - speechbrain.utils.train_logger - INFO - epoch: 28, lr_model: 3.52e-02, lr_wav2vec: 2.06e-05 - train loss: 5.55e-02 - valid loss: 5.01e-01, valid CER: 6.76, valid WER: 25.67
+2022-04-07 08:50:45,339 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+08-48-53+00
+2022-04-07 08:50:46,363 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+08-32-52+00
+2022-04-07 08:50:47,117 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+08-01-25+00
+2022-04-07 08:50:47,117 - speechbrain.utils.epoch_loop - INFO - Going into epoch 29
+2022-04-07 09:03:47,837 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.035 to 0.028
+2022-04-07 09:03:47,840 - speechbrain.nnet.schedulers - INFO - Changing lr from 2.1e-05 to 1.9e-05
+2022-04-07 09:03:47,887 - speechbrain.utils.train_logger - INFO - epoch: 29, lr_model: 3.52e-02, lr_wav2vec: 2.06e-05 - train loss: 5.44e-02 - valid loss: 5.10e-01, valid CER: 6.72, valid WER: 25.62
+2022-04-07 09:06:30,053 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+09-03-47+00
+2022-04-07 09:06:32,452 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+08-48-53+00
+2022-04-07 09:06:32,452 - speechbrain.utils.epoch_loop - INFO - Going into epoch 30
+2022-04-07 09:19:28,245 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.028 to 0.023
+2022-04-07 09:19:28,266 - speechbrain.nnet.schedulers - INFO - Changing lr from 1.9e-05 to 1.7e-05
+2022-04-07 09:19:28,342 - speechbrain.utils.train_logger - INFO - epoch: 30, lr_model: 2.81e-02, lr_wav2vec: 1.85e-05 - train loss: 5.01e-02 - valid loss: 5.29e-01, valid CER: 6.71, valid WER: 25.50
+2022-04-07 09:21:20,512 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+09-19-28+00
+2022-04-07 09:21:22,686 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+09-03-47+00
+2022-04-07 09:21:22,696 - speechbrain.utils.checkpoints - INFO - Loading a checkpoint from results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+09-19-28+00
+2022-04-07 09:21:42,982 - root - DEBUG - SaveableDataLoader was requested to load a checkpoint, but the DataLoader has already been iterated. The DataLoader file will be ignored. This is normal in evaluation, when a checkpoint is loaded just to retrieve the best model.
+2022-04-07 09:24:40,071 - speechbrain.utils.train_logger - INFO - Epoch loaded: 30 - test loss: 5.24e-01, test CER: 6.57, test WER: 24.92
+2022-04-07 09:25:00,438 - speechbrain.core - INFO - Beginning experiment!
+2022-04-07 09:25:00,447 - speechbrain.core - INFO - Experiment folder: results/wav2vec2_ctc_AMHARIC/1249
+2022-04-07 09:25:01,071 - speechbrain.utils.superpowers - DEBUG - aiohttp==3.8.1
+aiosignal==1.2.0
+appdirs==1.4.4
+async-timeout==4.0.2
+attrs==21.4.0
+audioread==2.1.9
+azure-core==1.21.1
+azure-storage-blob==12.9.0
+bcrypt==3.2.0
+black==19.10b0
+certifi==2021.10.8
+cffi==1.15.0
+cfgv==3.3.1
+charset-normalizer==2.0.10
+click==8.0.3
+cryptography==36.0.1
+datasets==1.13.3
+decorator==5.1.1
+dill==0.3.4
+distlib==0.3.4
+entrypoints==0.3
+ffmpeg==1.4
+filelock==3.4.2
+flake8==3.7.9
+frozenlist==1.3.0
+fsspec==2022.2.0
+huggingface-hub==0.2.1
+HyperPyYAML==1.0.0
+identify==2.4.4
+idna==3.3
+isodate==0.6.1
+joblib==1.1.0
+librosa==0.8.1
+llvmlite==0.38.0
+mccabe==0.6.1
+more-itertools==8.12.0
+msrest==0.6.21
+multidict==6.0.2
+multiprocess==0.70.12.2
+mutagen==1.45.1
+nodeenv==1.6.0
+numba==0.55.0
+numpy==1.21.5
+oauthlib==3.1.1
+packaging==21.3
+pandas==1.3.5
+paramiko==2.10.3
+pathspec==0.9.0
+platformdirs==2.4.1
+pluggy==0.13.1
+pooch==1.5.2
+pre-commit==2.17.0
+py==1.11.0
+pyarrow==7.0.0
+pycodestyle==2.5.0
+pycparser==2.21
+pyflakes==2.1.1
+PyNaCl==1.5.0
+pyparsing==3.0.6
+pytest==5.4.1
+python-dateutil==2.8.2
+pytz==2021.3
+PyYAML==6.0
+regex==2022.1.18
+requests==2.27.1
+requests-oauthlib==1.3.0
+resampy==0.2.2
+ruamel.yaml==0.17.20
+ruamel.yaml.clib==0.2.6
+sacremoses==0.0.47
+scikit-learn==1.0.2
+scipy==1.7.3
+scp==0.14.4
+sentencepiece==0.1.96
+six==1.16.0
+SoundFile==0.10.3.post1
+threadpoolctl==3.0.0
+tokenizers==0.10.3
+toml==0.10.2
+torch==1.10.1
+torchaudio==0.10.1
+tqdm==4.62.3
+transformers==4.13.0
+typed-ast==1.5.1
+typing_extensions==4.0.1
+urllib3==1.26.8
+virtualenv==20.13.0
+wcwidth==0.2.5
+xxhash==3.0.0
+yamllint==1.23.0
+yarl==1.7.2
+
+
+2022-04-07 09:25:01,407 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/train.csv already exists, skipping data preparation!
+2022-04-07 09:25:01,407 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/dev.csv already exists, skipping data preparation!
+2022-04-07 09:25:01,407 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/test.csv already exists, skipping data preparation!
+2022-04-07 09:25:01,413 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer is already trained.
+2022-04-07 09:25:01,413 - speechbrain.tokenizers.SentencePiece - INFO - ==== Loading Tokenizer ===
+2022-04-07 09:25:01,413 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer path: results/wav2vec2_ctc_AMHARIC/1249/save/224_char.model
+2022-04-07 09:25:01,413 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer vocab_size: 224
+2022-04-07 09:25:01,413 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer type: char
+2022-04-07 09:25:01,591 - speechbrain.core - INFO - Info: auto_mix_prec arg from hparam file is used
+2022-04-07 09:25:01,591 - speechbrain.core - INFO - Info: ckpt_interval_minutes arg from hparam file is used
+2022-04-07 09:25:05,095 - speechbrain.core - INFO - 318.8M trainable parameters in ASR
+2022-04-07 09:25:05,382 - speechbrain.utils.checkpoints - INFO - Loading a checkpoint from results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+09-19-28+00
+2022-04-07 09:25:08,583 - speechbrain.utils.checkpoints - INFO - Loading a checkpoint from results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+09-19-28+00
+2022-04-07 09:26:16,796 - speechbrain.utils.train_logger - INFO - Epoch loaded: 30 - test loss: 5.24e-01, test CER: 6.57, test WER: 24.92
+2022-05-27 08:17:50,151 - speechbrain.core - INFO - Beginning experiment!
+2022-05-27 08:17:50,151 - speechbrain.core - INFO - Experiment folder: results/wav2vec2_ctc_AMHARIC/1249
+2022-05-27 08:17:54,273 - speechbrain.utils.superpowers - DEBUG - aiohttp==3.8.1
+aiosignal==1.2.0
+appdirs==1.4.4
+async-timeout==4.0.2
+attrs==21.4.0
+audioread==2.1.9
+audiosegment==0.23.0
+azure-core==1.21.1
+azure-storage-blob==12.9.0
+bcrypt==3.2.0
+black==19.10b0
+certifi==2021.10.8
+cffi==1.15.0
+cfgv==3.3.1
+charset-normalizer==2.0.10
+click==8.0.3
+cryptography==36.0.1
+datasets==1.13.3
+decorator==5.1.1
+dill==0.3.4
+distlib==0.3.4
+entrypoints==0.3
+ffmpeg==1.4
+filelock==3.4.2
+flake8==3.7.9
+frozenlist==1.3.0
+fsspec==2022.2.0
+huggingface-hub==0.5.1
+HyperPyYAML==1.0.1
+identify==2.4.4
+idna==3.3
+isodate==0.6.1
+joblib==1.1.0
+librosa==0.8.1
+llvmlite==0.38.0
+mccabe==0.6.1
+more-itertools==8.12.0
+msrest==0.6.21
+multidict==6.0.2
+multiprocess==0.70.12.2
+mutagen==1.45.1
+nodeenv==1.6.0
+numba==0.55.0
+numpy==1.21.5
+oauthlib==3.1.1
+packaging==21.3
+pandas==1.3.5
+paramiko==2.10.3
+pathspec==0.9.0
+platformdirs==2.4.1
+pluggy==0.13.1
+pooch==1.5.2
+pre-commit==2.17.0
+py==1.11.0
+pyarrow==7.0.0
+pycodestyle==2.5.0
+pycparser==2.21
+pydub==0.25.1
+pyflakes==2.1.1
+PyNaCl==1.5.0
+pyparsing==3.0.6
+pytest==5.4.1
+python-dateutil==2.8.2
+pytz==2021.3
+PyYAML==6.0
+regex==2022.1.18
+requests==2.27.1
+requests-oauthlib==1.3.0
+resampy==0.2.2
+ruamel.yaml==0.17.21
+ruamel.yaml.clib==0.2.6
+sacremoses==0.0.53
+scikit-learn==1.0.2
+scipy==1.7.3
+scp==0.14.4
+sentencepiece==0.1.96
+six==1.16.0
+SoundFile==0.10.3.post1
+speechbrain==0.5.11
+threadpoolctl==3.0.0
+tokenizers==0.12.1
+toml==0.10.2
+torch==1.11.0
+torchaudio==0.11.0
+tqdm==4.62.3
+transformers==4.18.0
+typed-ast==1.5.1
+typing_extensions==4.0.1
+urllib3==1.26.8
+virtualenv==20.13.0
+wcwidth==0.2.5
+webrtcvad==2.0.10
+xxhash==3.0.0
+yamllint==1.23.0
+yarl==1.7.2
+youtube-dl==2021.12.17
+
+
+2022-05-27 08:19:01,150 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/train.csv already exists, skipping data preparation!
+2022-05-27 08:19:01,151 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/dev.csv already exists, skipping data preparation!
+2022-05-27 08:19:01,151 - dvoice_prepare - INFO - results/wav2vec2_ctc_AMHARIC/1249/save/test.csv already exists, skipping data preparation!
+2022-05-27 08:19:01,160 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer is already trained.
+2022-05-27 08:19:01,160 - speechbrain.tokenizers.SentencePiece - INFO - ==== Loading Tokenizer ===
+2022-05-27 08:19:01,160 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer path: results/wav2vec2_ctc_AMHARIC/1249/save/224_char.model
+2022-05-27 08:19:01,160 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer vocab_size: 224
+2022-05-27 08:19:01,160 - speechbrain.tokenizers.SentencePiece - INFO - Tokenizer type: char
+2022-05-27 08:19:01,425 - speechbrain.core - INFO - Info: auto_mix_prec arg from hparam file is used
+2022-05-27 08:19:01,425 - speechbrain.core - INFO - Info: ckpt_interval_minutes arg from hparam file is used
+2022-05-27 08:19:09,868 - speechbrain.core - INFO - 318.8M trainable parameters in ASR
+2022-05-27 08:19:09,899 - speechbrain.utils.checkpoints - INFO - Loading a checkpoint from results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+09-19-28+00
+2022-05-27 08:20:01,646 - speechbrain.utils.checkpoints - INFO - Loading a checkpoint from results/wav2vec2_ctc_AMHARIC/1249/save/CKPT+2022-04-07+09-19-28+00
+2022-05-27 08:20:59,252 - speechbrain.utils.train_logger - INFO - Epoch loaded: 30 - test loss: 5.24e-01, test CER: 6.57, test WER: 24.92

Training/train.py

@@ -0,0 +1,380 @@
+#!/usr/bin/env python3
+import sys
+
+
+
+sys.path
+sys.path.append('/data/n.abdoumohamed/DVoice/speechbrain')
+
+
+import torch
+import logging
+import speechbrain as sb
+import torchaudio
+from hyperpyyaml import load_hyperpyyaml
+from speechbrain.tokenizers.SentencePiece import SentencePiece
+from speechbrain.utils.data_utils import undo_padding
+from speechbrain.utils.distributed import run_on_main
+
+"""Recipe for training a sequence-to-sequence ASR system with CommonVoice.
+The system employs a wav2vec2 encoder and a CTC decoder.
+Decoding is performed with greedy decoding (will be extended to beam search).
+
+To run this recipe, do the following:
+> python train_with_wav2vec2.py hparams/train_with_wav2vec2.yaml
+
+With the default hyperparameters, the system employs a pretrained wav2vec2 encoder.
+The wav2vec2 model is pretrained following the model given in the hprams file.
+It may be dependent on the language.
+
+The neural network is trained with CTC on sub-word units estimated with
+Byte Pairwise Encoding (BPE).
+
+The experiment file is flexible enough to support a large variety of
+different systems. By properly changing the parameter files, you can try
+different encoders, decoders, tokens (e.g, characters instead of BPE),
+training languages (all CommonVoice languages), and many
+other possible variations.
+
+Authors
+ * Titouan Parcollet 2021
+"""
+
+logger = logging.getLogger(__name__)
+
+
+# Define training procedure
+class ASR(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        tokens_bos, _ = batch.tokens_bos
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        # Forward pass
+        feats = self.modules.wav2vec2(wavs)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return p_ctc, wav_lens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens = predictions
+
+        ids = batch.id
+        tokens_eos, tokens_eos_lens = batch.tokens_eos
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+        if stage != sb.Stage.TRAIN:
+            # Decode token terms to words
+            sequence = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+
+            predicted_words = self.tokenizer(sequence, task="decode_from_list")
+
+            # Convert indices to words
+            target_words = undo_padding(tokens, tokens_lens)
+            target_words = self.tokenizer(target_words, task="decode_from_list")
+
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        if self.auto_mix_prec:
+
+            if not self.hparams.wav2vec2.freeze:
+                self.wav2vec_optimizer.zero_grad()
+            self.model_optimizer.zero_grad()
+
+            with torch.cuda.amp.autocast():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            self.scaler.scale(loss).backward()
+            if not self.hparams.wav2vec2.freeze:
+                self.scaler.unscale_(self.wav2vec_optimizer)
+            self.scaler.unscale_(self.model_optimizer)
+
+            if self.check_gradients(loss):
+                if not self.hparams.wav2vec2.freeze:
+                    self.scaler.step(self.wav2vec_optimizer)
+                self.scaler.step(self.model_optimizer)
+
+            self.scaler.update()
+        else:
+            outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            loss.backward()
+
+            if self.check_gradients(loss):
+                if not self.hparams.wav2vec2.freeze:
+                    self.wav2vec_optimizer.step()
+                self.model_optimizer.step()
+
+            if not self.hparams.wav2vec2.freeze:
+                self.wav2vec_optimizer.zero_grad()
+            self.model_optimizer.zero_grad()
+
+        return loss.detach()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            if not self.hparams.wav2vec2.freeze:
+                sb.nnet.schedulers.update_learning_rate(
+                    self.wav2vec_optimizer, new_lr_wav2vec
+                )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                    "lr_wav2vec": old_lr_wav2vec,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+        "Initializes the wav2vec2 optimizer and model optimizer"
+
+        # If the wav2vec encoder is unfrozen, we create the optimizer
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(
+                self.modules.wav2vec2.parameters()
+            )
+            if self.checkpointer is not None:
+                self.checkpointer.add_recoverable(
+                    "wav2vec_opt", self.wav2vec_optimizer
+                )
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+
+# Define custom data procedure
+def dataio_prepare(hparams, tokenizer):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    # 1. Define datasets
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            reverse=True,
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"], replacements={"data_root": data_folder},
+    )
+    # We also sort the validation data so it is faster to validate
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+
+    test_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["test_csv"], replacements={"data_root": data_folder},
+    )
+
+    # We also sort the validation data so it is faster to validate
+    test_data = test_data.filtered_sorted(sort_key="duration")
+
+    datasets = [train_data, valid_data, test_data]
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        info = torchaudio.info(wav)
+        sig = sb.dataio.dataio.read_audio(wav)
+        resampled = torchaudio.transforms.Resample(
+            info.sample_rate, hparams["sample_rate"],
+        )(sig)
+        return resampled
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "tokens_list", "tokens_bos", "tokens_eos", "tokens"
+    )
+    def text_pipeline(wrd):
+        tokens_list = tokenizer.sp.encode_as_ids(wrd)
+        yield tokens_list
+        tokens_bos = torch.LongTensor([hparams["bos_index"]] + (tokens_list))
+        yield tokens_bos
+        tokens_eos = torch.LongTensor(tokens_list + [hparams["eos_index"]])
+        yield tokens_eos
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets, ["id", "sig", "tokens_bos", "tokens_eos", "tokens"],
+    )
+    return train_data, valid_data, test_data
+
+
+if __name__ == "__main__":
+
+    # Load hyperparameters file with command-line overrides
+    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+    with open(hparams_file) as fin:
+        hparams = load_hyperpyyaml(fin, overrides)
+
+    # If distributed_launch=True then
+    # create ddp_group with the right communication protocol
+    sb.utils.distributed.ddp_init_group(run_opts)
+
+    # Dataset preparation (parsing CommonVoice)
+    from dvoice_prepare import prepare_dvoice  # noqa
+
+    # Create experiment directory
+    sb.create_experiment_directory(
+        experiment_directory=hparams["output_folder"],
+        hyperparams_to_save=hparams_file,
+        overrides=overrides,
+    )
+
+    # Due to DDP, we do the preparation ONLY on the main python process
+    run_on_main(
+        prepare_dvoice,
+        kwargs={
+            "data_folder": hparams["data_folder"],
+            "save_folder": hparams["save_folder"],
+            "train_csv_file": hparams["train_csv_file"],
+            "dev_csv_file": hparams["dev_csv_file"],
+            "test_csv_file": hparams["test_csv_file"],
+            "accented_letters": hparams["accented_letters"],
+            "language": hparams["language"],
+            "skip_prep": hparams["skip_prep"],
+        },
+    )
+
+    # Defining tokenizer and loading it
+    tokenizer = SentencePiece(
+        model_dir=hparams["save_folder"],
+        vocab_size=hparams["output_neurons"],
+        annotation_train=hparams["train_csv"],
+        annotation_read="wrd",
+        model_type=hparams["token_type"],
+        character_coverage=hparams["character_coverage"],
+    )
+
+    # Create the datasets objects as well as tokenization and encoding :-D
+    train_data, valid_data, test_data = dataio_prepare(hparams, tokenizer)
+
+    # Trainer initialization
+    asr_brain = ASR(
+        modules=hparams["modules"],
+        hparams=hparams,
+        run_opts=run_opts,
+        checkpointer=hparams["checkpointer"],
+    )
+
+    # Adding objects to trainer.
+    asr_brain.tokenizer = tokenizer
+
+    # Training
+    asr_brain.fit(
+        asr_brain.hparams.epoch_counter,
+        train_data,
+        valid_data,
+        train_loader_kwargs=hparams["dataloader_options"],
+        valid_loader_kwargs=hparams["test_dataloader_options"],
+    )
+
+    # Test
+    asr_brain.hparams.wer_file = hparams["output_folder"] + "/wer_test.txt"
+    asr_brain.evaluate(
+        test_data,
+        min_key="WER",
+        test_loader_kwargs=hparams["test_dataloader_options"],
+    )

Training/train2.py

@@ -0,0 +1,372 @@
+import sys
+import torch
+import logging
+import speechbrain as sb
+import torchaudio
+from hyperpyyaml import load_hyperpyyaml
+from speechbrain.tokenizers.SentencePiece import SentencePiece
+from speechbrain.utils.data_utils import undo_padding
+from speechbrain.utils.distributed import run_on_main
+
+"""Recipe for training a sequence-to-sequence ASR system with CommonVoice.
+The system employs a wav2vec2 encoder and a CTC decoder.
+Decoding is performed with greedy decoding (will be extended to beam search).
+
+To run this recipe, do the following:
+> python train_with_wav2vec2.py hparams/train_with_wav2vec2.yaml
+
+With the default hyperparameters, the system employs a pretrained wav2vec2 encoder.
+The wav2vec2 model is pretrained following the model given in the hprams file.
+It may be dependent on the language.
+
+The neural network is trained with CTC on sub-word units estimated with
+Byte Pairwise Encoding (BPE).
+
+The experiment file is flexible enough to support a large variety of
+different systems. By properly changing the parameter files, you can try
+different encoders, decoders, tokens (e.g, characters instead of BPE),
+training languages (all CommonVoice languages), and many
+other possible variations.
+
+Authors
+ * Titouan Parcollet 2021
+"""
+
+logger = logging.getLogger(__name__)
+
+
+# Define training procedure
+class ASR(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        tokens_bos, _ = batch.tokens_bos
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        # Forward pass
+        feats = self.modules.wav2vec2(wavs)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return p_ctc, wav_lens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens = predictions
+
+        ids = batch.id
+        tokens_eos, tokens_eos_lens = batch.tokens_eos
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+        if stage != sb.Stage.TRAIN:
+            # Decode token terms to words
+            sequence = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+
+            predicted_words = self.tokenizer(sequence, task="decode_from_list")
+
+            # Convert indices to words
+            target_words = undo_padding(tokens, tokens_lens)
+            target_words = self.tokenizer(target_words, task="decode_from_list")
+
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        if self.auto_mix_prec:
+
+            if not self.hparams.wav2vec2.freeze:
+                self.wav2vec_optimizer.zero_grad()
+            self.model_optimizer.zero_grad()
+
+            with torch.cuda.amp.autocast():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            self.scaler.scale(loss).backward()
+            if not self.hparams.wav2vec2.freeze:
+                self.scaler.unscale_(self.wav2vec_optimizer)
+            self.scaler.unscale_(self.model_optimizer)
+
+            if self.check_gradients(loss):
+                if not self.hparams.wav2vec2.freeze:
+                    self.scaler.step(self.wav2vec_optimizer)
+                self.scaler.step(self.model_optimizer)
+
+            self.scaler.update()
+        else:
+            outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            loss.backward()
+
+            if self.check_gradients(loss):
+                if not self.hparams.wav2vec2.freeze:
+                    self.wav2vec_optimizer.step()
+                self.model_optimizer.step()
+
+            if not self.hparams.wav2vec2.freeze:
+                self.wav2vec_optimizer.zero_grad()
+            self.model_optimizer.zero_grad()
+
+        return loss.detach()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            if not self.hparams.wav2vec2.freeze:
+                sb.nnet.schedulers.update_learning_rate(
+                    self.wav2vec_optimizer, new_lr_wav2vec
+                )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                    "lr_wav2vec": old_lr_wav2vec,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+        "Initializes the wav2vec2 optimizer and model optimizer"
+
+        # If the wav2vec encoder is unfrozen, we create the optimizer
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(
+                self.modules.wav2vec2.parameters()
+            )
+            if self.checkpointer is not None:
+                self.checkpointer.add_recoverable(
+                    "wav2vec_opt", self.wav2vec_optimizer
+                )
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+
+# Define custom data procedure
+def dataio_prepare(hparams, tokenizer):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    # 1. Define datasets
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            reverse=True,
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"], replacements={"data_root": data_folder},
+    )
+    # We also sort the validation data so it is faster to validate
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+
+    test_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["test_csv"], replacements={"data_root": data_folder},
+    )
+
+    # We also sort the validation data so it is faster to validate
+    test_data = test_data.filtered_sorted(sort_key="duration")
+
+    datasets = [train_data, valid_data, test_data]
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        info = torchaudio.info(wav)
+        sig = sb.dataio.dataio.read_audio(wav)
+        resampled = torchaudio.transforms.Resample(
+            info.sample_rate, hparams["sample_rate"],
+        )(sig)
+        return resampled
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "tokens_list", "tokens_bos", "tokens_eos", "tokens"
+    )
+    def text_pipeline(wrd):
+        tokens_list = tokenizer.sp.encode_as_ids(wrd)
+        yield tokens_list
+        tokens_bos = torch.LongTensor([hparams["bos_index"]] + (tokens_list))
+        yield tokens_bos
+        tokens_eos = torch.LongTensor(tokens_list + [hparams["eos_index"]])
+        yield tokens_eos
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets, ["id", "sig", "tokens_bos", "tokens_eos", "tokens"],
+    )
+    return train_data, valid_data, test_data
+
+
+if __name__ == "__main__":
+
+    # Load hyperparameters file with command-line overrides
+    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+    with open(hparams_file) as fin:
+        hparams = load_hyperpyyaml(fin, overrides)
+
+    # If distributed_launch=True then
+    # create ddp_group with the right communication protocol
+    sb.utils.distributed.ddp_init_group(run_opts)
+
+    # Dataset preparation (parsing CommonVoice)
+    from dvoice_prepare import prepare_dvoice  # noqa
+
+    # Create experiment directory
+    sb.create_experiment_directory(
+        experiment_directory=hparams["output_folder"],
+        hyperparams_to_save=hparams_file,
+        overrides=overrides,
+    )
+
+    # Due to DDP, we do the preparation ONLY on the main python process
+    run_on_main(
+        prepare_dvoice,
+        kwargs={
+            "data_folder": hparams["data_folder"],
+            "save_folder": hparams["save_folder"],
+            "train_csv_file": hparams["train_csv_file"],
+            "dev_csv_file": hparams["dev_csv_file"],
+            "test_csv_file": hparams["test_csv_file"],
+            "accented_letters": hparams["accented_letters"],
+            "language": hparams["language"],
+            "skip_prep": hparams["skip_prep"],
+        },
+    )
+
+    # Defining tokenizer and loading it
+    tokenizer = SentencePiece(
+        model_dir=hparams["save_folder"],
+        vocab_size=hparams["output_neurons"],
+        annotation_train=hparams["train_csv"],
+        annotation_read="wrd",
+        model_type=hparams["token_type"],
+        character_coverage=hparams["character_coverage"],
+    )
+
+    # Create the datasets objects as well as tokenization and encoding :-D
+    train_data, valid_data, test_data = dataio_prepare(hparams, tokenizer)
+
+    # Trainer initialization
+    asr_brain = ASR(
+        modules=hparams["modules"],
+        hparams=hparams,
+        run_opts=run_opts,
+        checkpointer=hparams["checkpointer"],
+    )
+
+    # Adding objects to trainer.
+    asr_brain.tokenizer = tokenizer
+
+    # Training
+    asr_brain.fit(
+        asr_brain.hparams.epoch_counter,
+        train_data,
+        valid_data,
+        train_loader_kwargs=hparams["dataloader_options"],
+        valid_loader_kwargs=hparams["test_dataloader_options"],
+    )
+
+    # Test
+    asr_brain.hparams.wer_file = hparams["output_folder"] + "/wer_test.txt"
+    asr_brain.evaluate(
+        test_data,
+        min_key="WER",
+        test_loader_kwargs=hparams["test_dataloader_options"],
+    )

Training/train_log.txt

@@ -0,0 +1,33 @@
+epoch: 1, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 2.16 - valid loss: 6.31e-01, valid CER: 17.94, valid WER: 61.08
+epoch: 2, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 7.60e-01 - valid loss: 4.76e-01, valid CER: 13.59, valid WER: 49.70
+epoch: 3, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 5.88e-01 - valid loss: 4.04e-01, valid CER: 11.77, valid WER: 43.35
+epoch: 4, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 4.97e-01 - valid loss: 3.75e-01, valid CER: 10.61, valid WER: 39.29
+epoch: 5, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 4.29e-01 - valid loss: 3.71e-01, valid CER: 10.15, valid WER: 37.38
+epoch: 6, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 3.81e-01 - valid loss: 3.54e-01, valid CER: 9.54, valid WER: 35.23
+epoch: 7, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 3.39e-01 - valid loss: 3.41e-01, valid CER: 8.98, valid WER: 33.48
+epoch: 8, lr_model: 1.00e+00, lr_wav2vec: 1.00e-04 - train loss: 3.08e-01 - valid loss: 3.57e-01, valid CER: 8.80, valid WER: 32.41
+epoch: 9, lr_model: 8.00e-01, lr_wav2vec: 9.00e-05 - train loss: 2.70e-01 - valid loss: 3.46e-01, valid CER: 8.47, valid WER: 31.33
+epoch: 10, lr_model: 8.00e-01, lr_wav2vec: 9.00e-05 - train loss: 2.45e-01 - valid loss: 3.64e-01, valid CER: 8.30, valid WER: 30.31
+epoch: 11, lr_model: 6.40e-01, lr_wav2vec: 8.10e-05 - train loss: 2.17e-01 - valid loss: 3.43e-01, valid CER: 8.00, valid WER: 29.91
+epoch: 12, lr_model: 6.40e-01, lr_wav2vec: 8.10e-05 - train loss: 1.98e-01 - valid loss: 3.68e-01, valid CER: 7.93, valid WER: 29.49
+epoch: 13, lr_model: 5.12e-01, lr_wav2vec: 7.29e-05 - train loss: 1.75e-01 - valid loss: 3.94e-01, valid CER: 7.78, valid WER: 29.09
+epoch: 14, lr_model: 4.10e-01, lr_wav2vec: 6.56e-05 - train loss: 1.58e-01 - valid loss: 3.94e-01, valid CER: 7.75, valid WER: 28.90
+epoch: 15, lr_model: 3.28e-01, lr_wav2vec: 5.90e-05 - train loss: 1.37e-01 - valid loss: 4.13e-01, valid CER: 7.64, valid WER: 28.39
+epoch: 16, lr_model: 2.62e-01, lr_wav2vec: 5.31e-05 - train loss: 1.25e-01 - valid loss: 3.95e-01, valid CER: 7.48, valid WER: 27.94
+epoch: 17, lr_model: 2.62e-01, lr_wav2vec: 5.31e-05 - train loss: 1.20e-01 - valid loss: 4.12e-01, valid CER: 7.36, valid WER: 27.73
+epoch: 18, lr_model: 2.10e-01, lr_wav2vec: 4.78e-05 - train loss: 1.03e-01 - valid loss: 4.31e-01, valid CER: 7.44, valid WER: 27.69
+epoch: 19, lr_model: 1.68e-01, lr_wav2vec: 4.30e-05 - train loss: 9.82e-02 - valid loss: 4.35e-01, valid CER: 7.28, valid WER: 27.08
+epoch: 20, lr_model: 1.34e-01, lr_wav2vec: 3.87e-05 - train loss: 8.78e-02 - valid loss: 4.42e-01, valid CER: 7.27, valid WER: 27.18
+epoch: 21, lr_model: 1.07e-01, lr_wav2vec: 3.49e-05 - train loss: 8.27e-02 - valid loss: 4.72e-01, valid CER: 7.16, valid WER: 26.79
+epoch: 22, lr_model: 8.59e-02, lr_wav2vec: 3.14e-05 - train loss: 7.39e-02 - valid loss: 4.88e-01, valid CER: 7.03, valid WER: 26.28
+epoch: 23, lr_model: 6.87e-02, lr_wav2vec: 2.82e-05 - train loss: 7.24e-02 - valid loss: 4.92e-01, valid CER: 6.95, valid WER: 26.03
+epoch: 24, lr_model: 5.50e-02, lr_wav2vec: 2.54e-05 - train loss: 6.76e-02 - valid loss: 4.80e-01, valid CER: 6.89, valid WER: 26.11
+epoch: 25, lr_model: 5.50e-02, lr_wav2vec: 2.54e-05 - train loss: 6.21e-02 - valid loss: 5.01e-01, valid CER: 6.88, valid WER: 25.88
+epoch: 26, lr_model: 4.40e-02, lr_wav2vec: 2.29e-05 - train loss: 5.94e-02 - valid loss: 4.89e-01, valid CER: 6.90, valid WER: 26.08
+epoch: 27, lr_model: 4.40e-02, lr_wav2vec: 2.29e-05 - train loss: 5.54e-02 - valid loss: 5.09e-01, valid CER: 6.87, valid WER: 26.04
+epoch: 28, lr_model: 3.52e-02, lr_wav2vec: 2.06e-05 - train loss: 5.55e-02 - valid loss: 5.01e-01, valid CER: 6.76, valid WER: 25.67
+epoch: 29, lr_model: 3.52e-02, lr_wav2vec: 2.06e-05 - train loss: 5.44e-02 - valid loss: 5.10e-01, valid CER: 6.72, valid WER: 25.62
+epoch: 30, lr_model: 2.81e-02, lr_wav2vec: 1.85e-05 - train loss: 5.01e-02 - valid loss: 5.29e-01, valid CER: 6.71, valid WER: 25.50
+Epoch loaded: 30 - test loss: 5.24e-01, test CER: 6.57, test WER: 24.92
+Epoch loaded: 30 - test loss: 5.24e-01, test CER: 6.57, test WER: 24.92
+Epoch loaded: 30 - test loss: 5.24e-01, test CER: 6.57, test WER: 24.92

graphs.py

@@ -0,0 +1,36 @@
+import wandb
+import re
+
+# Initialize WandB project
+wandb.init(project="training-amharic-stt-visualizations", name="metrics_visualization")
+
+# Path to your training log file
+log_file = "Training/train_log.txt"
+
+# Function to parse logs
+# Function to parse logs
+def parse_logs(log_file):
+    """
+    Parses the training logs and yields metrics as dictionaries.
+    """
+    with open(log_file, "r") as f:
+        for line in f:
+            # Match the log format using regex
+            match = re.match(
+                r"epoch: (?P<epoch>\d+), lr_model: (?P<lr_model>[0-9.e+-]+), lr_wav2vec: (?P<lr_wav2vec>[0-9.e+-]+) - "
+                r"train loss: (?P<train_loss>[0-9.e+-]+) - valid loss: (?P<valid_loss>[0-9.e+-]+), "
+                r"valid CER: (?P<valid_CER>[0-9.e+-]+), valid WER: (?P<valid_WER>[0-9.e+-]+)",
+                line.strip()
+            )
+            if match:
+                metrics = {key: float(value) if '.' in value or 'e' in value else int(value)
+                           for key, value in match.groupdict().items()}
+                yield metrics
+
+# Parse logs and log to WandB
+for metrics in parse_logs(log_file):
+    # Log metrics to WandB
+    wandb.log(metrics)
+
+# Finish WandB run
+wandb.finish()