Training in progress, step 250

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1d08c9740a6b886a8f3440244ee7d894c555c8ee7c64898e14fb64ab76ed829
+oid sha256:c56d8171c7e99a5c2a9e54c0c4428bdc8901de1a9fc1f2248238850b2cae028f
 size 1262104049

run.sh

@@ -8,17 +8,17 @@ python run_speech_recognition_ctc_bnb.py \
         --output_dir="./" \
         --overwrite_output_dir \
         --hub_model_id="NbAiLab/wav2vec2-xlsr-300m-norwegian2" \
-        --num_train_epochs="50" \
-        --per_device_train_batch_size="16" \
-        --per_device_eval_batch_size="16" \
+        --num_train_epochs="100" \
+        --per_device_train_batch_size="8" \
+        --per_device_eval_batch_size="8" \
         --gradient_accumulation_steps="4" \
         --learning_rate="7.5e-5" \
         --warmup_steps="2000" \
         --evaluation_strategy="steps" \
 	--length_column_name="input_length" \
         --chars_to_ignore , ? . ! \- \; \: \" " % ‘ " � — ’ … – \
-        --save_steps="100" \
-        --eval_steps="100" \
+        --save_steps="250" \
+        --eval_steps="250" \
         --logging_steps="100" \
         --layerdrop="0.0" \
         --activation_dropout="0.1" \

run_speech_recognition_ctc_bnb.py

@@ -686,25 +686,25 @@ def main():
     # Instantiate custom data collator
     data_collator = DataCollatorCTCWithPadding(processor=processor)
 
-    decay_parameters = get_parameter_names(model, [torch.nn.LayerNorm])
-    decay_parameters = [name for name in decay_parameters if "bias" not in name]
-    optimizer_grouped_parameters = [
-        {
-            "params": [p for n, p in model.named_parameters() if n in decay_parameters],
-            "weight_decay": training_args.weight_decay,
-        },
-        {
-            "params": [p for n, p in model.named_parameters() if n not in decay_parameters],
-            "weight_decay": 0.0,
-        },
-    ]
-    optimizer = bnb.optim.Adam8bit(
-        params=optimizer_grouped_parameters,
-        betas=(training_args.adam_beta1, training_args.adam_beta2),
-        eps=training_args.adam_epsilon,
-    )
-
-    optimizers = (optimizer, None)
+    #decay_parameters = get_parameter_names(model, [torch.nn.LayerNorm])
+    #decay_parameters = [name for name in decay_parameters if "bias" not in name]
+    #optimizer_grouped_parameters = [
+    #    {
+    #        "params": [p for n, p in model.named_parameters() if n in decay_parameters],
+    #        "weight_decay": training_args.weight_decay,
+    #    },
+    #    {
+    #        "params": [p for n, p in model.named_parameters() if n not in decay_parameters],
+    #        "weight_decay": 0.0,
+    #    },
+    #]
+    #optimizer = bnb.optim.Adam8bit(
+    #    params=optimizer_grouped_parameters,
+    #    betas=(training_args.adam_beta1, training_args.adam_beta2),
+    #    eps=training_args.adam_epsilon,
+    #)
+
+    #optimizers = (optimizer, None)
 
     # Initialize Trainer
     trainer = Trainer(
@@ -715,8 +715,8 @@ def main():
         train_dataset=vectorized_datasets["train"] if training_args.do_train else None,
         eval_dataset=vectorized_datasets["eval"] if training_args.do_eval else None,
         tokenizer=feature_extractor,
-        optimizers=optimizers,
-    )
+        #optimizers=optimizers,
+        )
 
     # 8. Finally, we can start training
 

runs/Jan28_13-50-07_job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba/events.out.tfevents.1643378396.job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba.124745.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:969c3f5031c2e78d325e0766501a0f980394cd5e2824976618fcda868f808e28
-size 16585
+oid sha256:2abb1d86b00c8a06e558229507420c951af09afb063ee92e8eb4f72c8b2a4d9c
+size 18010

runs/Jan28_19-16-02_job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba/1643397461.831888/events.out.tfevents.1643397461.job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba.261029.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22295f46dbc159d371cb3f6f7811ecb0a43041a21bd080154de37696780d1096
+size 4786

runs/Jan28_19-16-02_job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba/events.out.tfevents.1643397461.job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba.261029.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9193521f5ce2e3c7e6b144d8d2e145fe762c0966a69c1e47edb42c648ad67bc5
+size 5347

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b0526ca3463f8ec74dc67da7620f73cd644133f9228ca70552a449cd4c1f1ee
+oid sha256:249302cc973fa88a60488bebc95f0dde19627b173724b2e0910323ff0a12fe01
 size 3055