Training in progress, step 50

{checkpoint-40 → checkpoint-100}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6dd2f6e11f8931247aaf820f2fb0419ac8040fa59b8ef05119421d1d540f17c
+oid sha256:e1722a491ea58423692abe6cde8aac0703fa17d8346a9ae868114f2509e04c97
 size 2490337361

{checkpoint-60 → checkpoint-100}/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed09bae03da69e6e7d54c45ff801a7cd8e65f5bb600585b021024798e5fafa4a
+oid sha256:7998265fd0c18bea1f246ca75cca77edd2def33646432a9437b4d839e97eab3d
 size 1262063089

{checkpoint-60 → checkpoint-100}/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d5db96c9803a1c8f4051b38118bfa8389597dfc7915de251da9a5e1c69d3bc9
-size 14567
+oid sha256:24a8cdd3a3488a3315f9617b64f16798a9e655a6d49b281bfbe024d5e3f2a880
+size 14503

{checkpoint-60 → checkpoint-100}/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81ed4ec308038905cbb975b9b2ae3c7852901f7a7bb341d7393b61de64cbb13f
+oid sha256:2f28dfdeb34cb1b60f5d9c9b44dc5523bf2866ea557da6b58b27104a0030e0b3
 size 559

{checkpoint-40 → checkpoint-100}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c14d79112c3d0bf1ec781a02506228c0a8892e541191602a074e77b077eb126d
+oid sha256:6af49a56cc0b3dfaf6c4d29d3b9ab21c908c0ad2d95e41f19e6b8101ed863a39
 size 623

{checkpoint-60 → checkpoint-100}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.7947019867549668,
-  "global_step": 60,
+  "epoch": 1.3311258278145695,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -69,11 +69,53 @@
       "eval_steps_per_second": 0.814,
       "eval_wer": 1.0,
       "step": 60
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.959183673469387e-05,
+      "loss": 3.7607,
+      "step": 70
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 6.857142857142857e-05,
+      "loss": 3.638,
+      "step": 80
+    },
+    {
+      "epoch": 1.07,
+      "eval_loss": 3.4979612827301025,
+      "eval_runtime": 5.0758,
+      "eval_samples_per_second": 19.307,
+      "eval_steps_per_second": 0.788,
+      "eval_wer": 1.0,
+      "step": 80
+    },
+    {
+      "epoch": 1.2,
+      "learning_rate": 6.755102040816325e-05,
+      "loss": 3.4601,
+      "step": 90
+    },
+    {
+      "epoch": 1.33,
+      "learning_rate": 6.653061224489796e-05,
+      "loss": 3.4138,
+      "step": 100
+    },
+    {
+      "epoch": 1.33,
+      "eval_loss": 3.489065170288086,
+      "eval_runtime": 4.9074,
+      "eval_samples_per_second": 19.97,
+      "eval_steps_per_second": 0.815,
+      "eval_wer": 1.0,
+      "step": 100
     }
   ],
   "max_steps": 750,
   "num_train_epochs": 10,
-  "total_flos": 1.21965615465984e+18,
+  "total_flos": 2.00880445675488e+18,
   "trial_name": null,
   "trial_params": null
 }

checkpoint-40/trainer_state.json

@@ -1,58 +0,0 @@
-{
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 0.5298013245033113,
-  "global_step": 40,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.13,
-      "learning_rate": 4.9999999999999996e-05,
-      "loss": 13.2048,
-      "step": 10
-    },
-    {
-      "epoch": 0.26,
-      "learning_rate": 7.46938775510204e-05,
-      "loss": 26.5657,
-      "step": 20
-    },
-    {
-      "epoch": 0.26,
-      "eval_loss": 17.199691772460938,
-      "eval_runtime": 5.248,
-      "eval_samples_per_second": 18.674,
-      "eval_steps_per_second": 0.762,
-      "eval_wer": 1.0,
-      "step": 20
-    },
-    {
-      "epoch": 0.4,
-      "learning_rate": 7.36734693877551e-05,
-      "loss": 8.0091,
-      "step": 30
-    },
-    {
-      "epoch": 0.53,
-      "learning_rate": 7.265306122448979e-05,
-      "loss": 7.6332,
-      "step": 40
-    },
-    {
-      "epoch": 0.53,
-      "eval_loss": 4.783994197845459,
-      "eval_runtime": 4.9818,
-      "eval_samples_per_second": 19.672,
-      "eval_steps_per_second": 0.803,
-      "eval_wer": 1.0,
-      "step": 40
-    }
-  ],
-  "max_steps": 750,
-  "num_train_epochs": 10,
-  "total_flos": 8.2206860007168e+17,
-  "trial_name": null,
-  "trial_params": null
-}

{checkpoint-60 → checkpoint-50}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d06df765e9a7d7ea6615404de49841893d9b9ce55d42f0bfd5486fd7105d83c4
+oid sha256:4f8c3352ebf31a7c0483167400d3399b22ccc2be15a1cf307f0d154c758e9499
 size 2490337361

{checkpoint-40 → checkpoint-50}/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7825b23117146d129b9fbfae78e758d25f4f6e94f91b7f4977dd069a0f502f81
+oid sha256:80ebb6ebc7a95b13fda5cdc22fab5fcfb9ffdb99ca0102065a7147116a7f9f3f
 size 1262063089

{checkpoint-40 → checkpoint-50}/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9dd2cfddafbb139c577d6100cda2fcc2338327f63d6bca8b79767529c7f40e74
-size 14567
+oid sha256:2fe8f634109af89a7e3706d16f2fb93f152eb68502ebee688d78216511e43324
+size 14503

{checkpoint-40 → checkpoint-50}/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf412e9944a68f26d1088ddec860c9d22492bf7207573137d3159a8ac457e674
+oid sha256:83a7ca717969781e5bd7f66c4b808307bbfaef76d5ae72d82fe31d60dea27fb7
 size 559

{checkpoint-60 → checkpoint-50}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d173dd827ce26f584cfa95a40de1490d99e5852c0a8f273527eda424dd24205a
+oid sha256:fb5336e922700acd511fdefd5fbbe9ccf91f59901dbca8c97f53c0892943c4b7
 size 623

checkpoint-50/trainer_state.json

@@ -0,0 +1,37 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.6389776357827476,
+  "global_step": 50,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0007307692307692308,
+      "loss": 4.2559,
+      "step": 20
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0007115384615384615,
+      "loss": 3.068,
+      "step": 40
+    },
+    {
+      "epoch": 0.64,
+      "eval_loss": 3.058549165725708,
+      "eval_runtime": 3.3353,
+      "eval_samples_per_second": 29.982,
+      "eval_steps_per_second": 1.199,
+      "eval_wer": 1.0,
+      "step": 50
+    }
+  ],
+  "max_steps": 780,
+  "num_train_epochs": 10,
+  "total_flos": 7.755792033779712e+17,
+  "trial_name": null,
+  "trial_params": null
+}

{checkpoint-60 → checkpoint-50}/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:470abf25a211ef5ec3a9821c884f2749839bee5e95f633b168d26f90c1809409
+oid sha256:31becb5e425437ea470512000be6c31bb502b71fe15942ff8cf0cc01eb8267a0
 size 3055

old_run.sh

@@ -10,15 +10,15 @@ python old_run_asr_ctc.py \
 	--per_device_train_batch_size="32" \
 	--per_device_eval_batch_size="32" \
 	--gradient_accumulation_steps="4" \
-	--learning_rate="7.5e-5" \
+	--learning_rate="7.5e-4" \
     --max_train_samples="10000" \
     --max_eval_samples="100" \
 	--warmup_ratio="0.0" \
 	--length_column_name="input_length" \
 	--evaluation_strategy="steps" \
 	--save_strategy="steps" \
-	--eval_steps="100" \
-	--save_steps="100" \
+	--eval_steps="50" \
+	--save_steps="50" \
 	--text_column_name="sentence" \
 	--chars_to_ignore , ? . ! \- \; \: \" “ % ‘ ” � — ’ … – \
 	--logging_steps="20" \

old_run_asr_ctc.py

@@ -783,13 +783,17 @@ def main():
     # If dataset_seed is set, shuffle train
     if data_args.dataset_seed is not None:
         vectorized_datasets["train"] = vectorized_datasets["train"].shuffle(seed=data_args.dataset_seed)
-    
+
     # TODO: Log sample of datasets in the right way (see wandb docs)
     pd_train = vectorized_datasets["train"].select(range(10)).to_pandas()
     pd_eval = vectorized_datasets["eval"].select(range(10)).to_pandas()
     # wandb.log({"train_sample": pd_train})
     # wandb.log({"eval_sample": pd_eval})
 
+    print(pd_train)
+    print(pd_eval)
+
+
     # for large datasets it is advised to run the preprocessing on a
     # single machine first with ``args.preprocessing_only`` since there will mostly likely
     # be a timeout when running the script in distributed mode.
@@ -806,9 +810,12 @@ def main():
         pred.label_ids[pred.label_ids == -100] = tokenizer.pad_token_id
 
         pred_str = tokenizer.batch_decode(pred_ids)
+
         # we do not want to group tokens when computing the metrics
         label_str = tokenizer.batch_decode(pred.label_ids, group_tokens=False)
 
+        print(pred_str)
+        print(label_str)
         metrics = {k: v.compute(predictions=pred_str, references=label_str) for k, v in eval_metrics.items()}
 
         return metrics

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d243142fcbd07def1347c28af05f32d03aa37c7536de513cc8503d261cf6d773
+oid sha256:80ebb6ebc7a95b13fda5cdc22fab5fcfb9ffdb99ca0102065a7147116a7f9f3f
 size 1262063089

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:470abf25a211ef5ec3a9821c884f2749839bee5e95f633b168d26f90c1809409
+oid sha256:31becb5e425437ea470512000be6c31bb502b71fe15942ff8cf0cc01eb8267a0
 size 3055