End of training

README.md

@@ -3,6 +3,9 @@ library_name: transformers
 license: cc-by-nc-4.0
 base_model: facebook/mms-1b-all
 tags:
+- automatic-speech-recognition
+- bigcgen
+- mms
 - generated_from_trainer
 metrics:
 - wer
@@ -16,10 +19,10 @@ should probably proofread and complete it, then remove this comment. -->
 
 # mms-1b-bigcgen-combined-30hrs-model
 
-This model is a fine-tuned version of [facebook/mms-1b-all](https://huggingface.co/facebook/mms-1b-all) on an unknown dataset.
+This model is a fine-tuned version of [facebook/mms-1b-all](https://huggingface.co/facebook/mms-1b-all) on the BIGCGEN - BEM dataset.
 It achieves the following results on the evaluation set:
 - Loss: inf
-- Wer: 0.5079
+- Wer: 0.5077
 
 ## Model description
 

adapter.bem.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8610701dfe2addbc1a4caf3e28aa27a1a6f84c18f45dedf0a2a2c1b4e8c9f0d
+oid sha256:d3c87152ca5aeef15eaec8be18dbc8a9720c71c5afffa7c45df5f268609623ee
 size 8793408

all_results.json

@@ -1,15 +1,15 @@
 {
-    "epoch": 0.2035105571101501,
+    "epoch": 1.2716865937420503,
     "eval_loss": Infinity,
-    "eval_runtime": 64.2058,
+    "eval_runtime": 64.0032,
     "eval_samples": 916,
-    "eval_samples_per_second": 14.267,
-    "eval_steps_per_second": 3.567,
-    "eval_wer": 0.5658570761326482,
-    "total_flos": 2.0370481464218854e+18,
-    "train_loss": 6.35770435333252,
-    "train_runtime": 812.3434,
+    "eval_samples_per_second": 14.312,
+    "eval_steps_per_second": 3.578,
+    "eval_wer": 0.5077066791219057,
+    "total_flos": 1.2839004390911662e+19,
+    "train_loss": 2.4182849487304687,
+    "train_runtime": 5081.644,
     "train_samples": 15721,
-    "train_samples_per_second": 580.58,
-    "train_steps_per_second": 72.568
+    "train_samples_per_second": 3.936,
+    "train_steps_per_second": 0.492
 }

eval_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.2035105571101501,
+    "epoch": 1.2716865937420503,
     "eval_loss": Infinity,
-    "eval_runtime": 64.2058,
+    "eval_runtime": 64.0032,
     "eval_samples": 916,
-    "eval_samples_per_second": 14.267,
-    "eval_steps_per_second": 3.567,
-    "eval_wer": 0.5658570761326482
+    "eval_samples_per_second": 14.312,
+    "eval_steps_per_second": 3.578,
+    "eval_wer": 0.5077066791219057
 }

runs/Dec29_05-05-20_srvrocgpu011.uct.ac.za/events.out.tfevents.1735447124.srvrocgpu011.uct.ac.za

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f541259847c72ab7cbf1c6b506aee80245fea2f443ecb9d628e39046b6319a6
+size 40

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.2035105571101501,
-    "total_flos": 2.0370481464218854e+18,
-    "train_loss": 6.35770435333252,
-    "train_runtime": 812.3434,
+    "epoch": 1.2716865937420503,
+    "total_flos": 1.2839004390911662e+19,
+    "train_loss": 2.4182849487304687,
+    "train_runtime": 5081.644,
     "train_samples": 15721,
-    "train_samples_per_second": 580.58,
-    "train_steps_per_second": 72.568
+    "train_samples_per_second": 3.936,
+    "train_steps_per_second": 0.492
 }

trainer_state.json

@@ -1,102 +1,429 @@
 {
   "best_metric": Infinity,
   "best_model_checkpoint": null,
-  "epoch": 0.2035105571101501,
+  "epoch": 1.2716865937420503,
   "eval_steps": 100,
-  "global_step": 400,
+  "global_step": 2500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.05087763927753752,
-      "grad_norm": 6.116684436798096,
+      "grad_norm": 6.590461730957031,
       "learning_rate": 0.00028799999999999995,
-      "loss": 14.5999,
+      "loss": 14.5866,
       "step": 100
     },
     {
       "epoch": 0.05087763927753752,
       "eval_loss": Infinity,
-      "eval_runtime": 64.8429,
-      "eval_samples_per_second": 14.126,
-      "eval_steps_per_second": 3.532,
-      "eval_wer": 1.0224194301728164,
+      "eval_runtime": 64.3651,
+      "eval_samples_per_second": 14.231,
+      "eval_steps_per_second": 3.558,
+      "eval_wer": 1.0166978047641289,
       "step": 100
     },
     {
       "epoch": 0.10175527855507505,
-      "grad_norm": 7.5614237785339355,
-      "learning_rate": 0.0002995106202209006,
-      "loss": 6.1213,
+      "grad_norm": 6.888458728790283,
+      "learning_rate": 0.00028799999999999995,
+      "loss": 6.2466,
       "step": 200
     },
     {
       "epoch": 0.10175527855507505,
       "eval_loss": Infinity,
-      "eval_runtime": 64.1763,
-      "eval_samples_per_second": 14.273,
-      "eval_steps_per_second": 3.568,
-      "eval_wer": 1.0131947687996263,
+      "eval_runtime": 63.9312,
+      "eval_samples_per_second": 14.328,
+      "eval_steps_per_second": 3.582,
+      "eval_wer": 1.0018682858477348,
       "step": 200
     },
     {
       "epoch": 0.15263291783261257,
-      "grad_norm": 7.122040748596191,
-      "learning_rate": 0.000299000849617672,
-      "loss": 2.9652,
+      "grad_norm": 8.405721664428711,
+      "learning_rate": 0.0002755,
+      "loss": 5.4142,
       "step": 300
     },
     {
       "epoch": 0.15263291783261257,
       "eval_loss": Infinity,
-      "eval_runtime": 64.2436,
-      "eval_samples_per_second": 14.258,
-      "eval_steps_per_second": 3.565,
-      "eval_wer": 0.5965670247547875,
+      "eval_runtime": 63.7638,
+      "eval_samples_per_second": 14.366,
+      "eval_steps_per_second": 3.591,
+      "eval_wer": 0.9912424100887436,
       "step": 300
     },
     {
       "epoch": 0.2035105571101501,
-      "grad_norm": 2.8880279064178467,
-      "learning_rate": 0.0002984910790144435,
-      "loss": 1.7444,
+      "grad_norm": 4.623055458068848,
+      "learning_rate": 0.000263,
+      "loss": 2.1374,
       "step": 400
     },
     {
       "epoch": 0.2035105571101501,
       "eval_loss": Infinity,
-      "eval_runtime": 64.5642,
-      "eval_samples_per_second": 14.187,
-      "eval_steps_per_second": 3.547,
-      "eval_wer": 0.5657403082671649,
+      "eval_runtime": 64.3132,
+      "eval_samples_per_second": 14.243,
+      "eval_steps_per_second": 3.561,
+      "eval_wer": 0.5951658103689864,
       "step": 400
     },
     {
-      "epoch": 0.2035105571101501,
-      "step": 400,
-      "total_flos": 2.0370481464218854e+18,
-      "train_loss": 6.35770435333252,
-      "train_runtime": 812.3434,
-      "train_samples_per_second": 580.58,
-      "train_steps_per_second": 72.568
+      "epoch": 0.2543881963876876,
+      "grad_norm": 4.835407257080078,
+      "learning_rate": 0.00025049999999999996,
+      "loss": 1.741,
+      "step": 500
+    },
+    {
+      "epoch": 0.2543881963876876,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.5742,
+      "eval_samples_per_second": 14.185,
+      "eval_steps_per_second": 3.546,
+      "eval_wer": 0.564105558150397,
+      "step": 500
+    },
+    {
+      "epoch": 0.30526583566522514,
+      "grad_norm": 3.17110276222229,
+      "learning_rate": 0.00023799999999999998,
+      "loss": 1.6543,
+      "step": 600
+    },
+    {
+      "epoch": 0.30526583566522514,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.0009,
+      "eval_samples_per_second": 14.312,
+      "eval_steps_per_second": 3.578,
+      "eval_wer": 0.5607192900513779,
+      "step": 600
+    },
+    {
+      "epoch": 0.35614347494276266,
+      "grad_norm": 3.9509639739990234,
+      "learning_rate": 0.00022549999999999998,
+      "loss": 1.6579,
+      "step": 700
+    },
+    {
+      "epoch": 0.35614347494276266,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.787,
+      "eval_samples_per_second": 14.139,
+      "eval_steps_per_second": 3.535,
+      "eval_wer": 0.5585007006071929,
+      "step": 700
+    },
+    {
+      "epoch": 0.4070211142203002,
+      "grad_norm": 4.748206615447998,
+      "learning_rate": 0.00021299999999999997,
+      "loss": 1.676,
+      "step": 800
+    },
+    {
+      "epoch": 0.4070211142203002,
+      "eval_loss": Infinity,
+      "eval_runtime": 65.1489,
+      "eval_samples_per_second": 14.06,
+      "eval_steps_per_second": 3.515,
+      "eval_wer": 0.5475245212517516,
+      "step": 800
+    },
+    {
+      "epoch": 0.4578987534978377,
+      "grad_norm": 4.590206146240234,
+      "learning_rate": 0.0002005,
+      "loss": 1.5245,
+      "step": 900
+    },
+    {
+      "epoch": 0.4578987534978377,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.8036,
+      "eval_samples_per_second": 14.135,
+      "eval_steps_per_second": 3.534,
+      "eval_wer": 0.5409855207846801,
+      "step": 900
+    },
+    {
+      "epoch": 0.5087763927753752,
+      "grad_norm": 3.051598072052002,
+      "learning_rate": 0.000188,
+      "loss": 1.6324,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5087763927753752,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.4656,
+      "eval_samples_per_second": 14.209,
+      "eval_steps_per_second": 3.552,
+      "eval_wer": 0.5277907519850538,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5596540320529128,
+      "grad_norm": 3.247791051864624,
+      "learning_rate": 0.00017549999999999998,
+      "loss": 1.6878,
+      "step": 1100
+    },
+    {
+      "epoch": 0.5596540320529128,
+      "eval_loss": Infinity,
+      "eval_runtime": 65.077,
+      "eval_samples_per_second": 14.076,
+      "eval_steps_per_second": 3.519,
+      "eval_wer": 0.5244044838860346,
+      "step": 1100
+    },
+    {
+      "epoch": 0.6105316713304503,
+      "grad_norm": 3.290109395980835,
+      "learning_rate": 0.00016299999999999998,
+      "loss": 1.4994,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6105316713304503,
+      "eval_loss": Infinity,
+      "eval_runtime": 65.091,
+      "eval_samples_per_second": 14.073,
+      "eval_steps_per_second": 3.518,
+      "eval_wer": 0.525922466137319,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6614093106079878,
+      "grad_norm": 3.3608410358428955,
+      "learning_rate": 0.0001505,
+      "loss": 1.544,
+      "step": 1300
+    },
+    {
+      "epoch": 0.6614093106079878,
+      "eval_loss": Infinity,
+      "eval_runtime": 65.4332,
+      "eval_samples_per_second": 13.999,
+      "eval_steps_per_second": 3.5,
+      "eval_wer": 0.5211349836524989,
+      "step": 1300
+    },
+    {
+      "epoch": 0.7122869498855253,
+      "grad_norm": 6.465691089630127,
+      "learning_rate": 0.000138,
+      "loss": 1.5796,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7122869498855253,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.5408,
+      "eval_samples_per_second": 14.193,
+      "eval_steps_per_second": 3.548,
+      "eval_wer": 0.5244044838860346,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7631645891630628,
+      "grad_norm": 3.001516819000244,
+      "learning_rate": 0.0001255,
+      "loss": 1.3625,
+      "step": 1500
+    },
+    {
+      "epoch": 0.7631645891630628,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.707,
+      "eval_samples_per_second": 14.156,
+      "eval_steps_per_second": 3.539,
+      "eval_wer": 0.5234703409621672,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8140422284406004,
+      "grad_norm": 4.548780918121338,
+      "learning_rate": 0.00011312499999999999,
+      "loss": 1.4826,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8140422284406004,
+      "eval_loss": Infinity,
+      "eval_runtime": 65.203,
+      "eval_samples_per_second": 14.048,
+      "eval_steps_per_second": 3.512,
+      "eval_wer": 0.5164642690331621,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8649198677181379,
+      "grad_norm": 2.6502480506896973,
+      "learning_rate": 0.00010062499999999998,
+      "loss": 1.4439,
+      "step": 1700
+    },
+    {
+      "epoch": 0.8649198677181379,
+      "eval_loss": Infinity,
+      "eval_runtime": 65.2704,
+      "eval_samples_per_second": 14.034,
+      "eval_steps_per_second": 3.508,
+      "eval_wer": 0.5226529659037833,
+      "step": 1700
+    },
+    {
+      "epoch": 0.9157975069956754,
+      "grad_norm": 10.055949211120605,
+      "learning_rate": 8.8125e-05,
+      "loss": 1.4778,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9157975069956754,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.7369,
+      "eval_samples_per_second": 14.15,
+      "eval_steps_per_second": 3.537,
+      "eval_wer": 0.5148295189163942,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9666751462732129,
+      "grad_norm": 2.887960433959961,
+      "learning_rate": 7.5625e-05,
+      "loss": 1.389,
+      "step": 1900
+    },
+    {
+      "epoch": 0.9666751462732129,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.8495,
+      "eval_samples_per_second": 14.125,
+      "eval_steps_per_second": 3.531,
+      "eval_wer": 0.5129612330686595,
+      "step": 1900
+    },
+    {
+      "epoch": 1.0172983973543628,
+      "grad_norm": 2.467592716217041,
+      "learning_rate": 6.312499999999999e-05,
+      "loss": 1.3863,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0172983973543628,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.9736,
+      "eval_samples_per_second": 14.098,
+      "eval_steps_per_second": 3.525,
+      "eval_wer": 0.5177487155534797,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0681760366319002,
+      "grad_norm": 2.5497734546661377,
+      "learning_rate": 5.0625e-05,
+      "loss": 1.516,
+      "step": 2100
+    },
+    {
+      "epoch": 1.0681760366319002,
+      "eval_loss": Infinity,
+      "eval_runtime": 65.4577,
+      "eval_samples_per_second": 13.994,
+      "eval_steps_per_second": 3.498,
+      "eval_wer": 0.5081737505838393,
+      "step": 2100
+    },
+    {
+      "epoch": 1.1190536759094378,
+      "grad_norm": 2.2054882049560547,
+      "learning_rate": 3.812499999999999e-05,
+      "loss": 1.474,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1190536759094378,
+      "eval_loss": Infinity,
+      "eval_runtime": 65.1492,
+      "eval_samples_per_second": 14.06,
+      "eval_steps_per_second": 3.515,
+      "eval_wer": 0.5106258757589911,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1699313151869752,
+      "grad_norm": 4.596852779388428,
+      "learning_rate": 2.5625e-05,
+      "loss": 1.465,
+      "step": 2300
+    },
+    {
+      "epoch": 1.1699313151869752,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.8751,
+      "eval_samples_per_second": 14.119,
+      "eval_steps_per_second": 3.53,
+      "eval_wer": 0.5077066791219057,
+      "step": 2300
+    },
+    {
+      "epoch": 1.2208089544645129,
+      "grad_norm": 2.048726797103882,
+      "learning_rate": 1.3124999999999999e-05,
+      "loss": 1.484,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2208089544645129,
+      "eval_loss": Infinity,
+      "eval_runtime": 64.9644,
+      "eval_samples_per_second": 14.1,
+      "eval_steps_per_second": 3.525,
+      "eval_wer": 0.5089911256422233,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2716865937420503,
+      "grad_norm": 1.6926039457321167,
+      "learning_rate": 7.499999999999999e-07,
+      "loss": 1.3942,
+      "step": 2500
+    },
+    {
+      "epoch": 1.2716865937420503,
+      "eval_loss": Infinity,
+      "eval_runtime": 65.9732,
+      "eval_samples_per_second": 13.884,
+      "eval_steps_per_second": 3.471,
+      "eval_wer": 0.5079402148528724,
+      "step": 2500
+    },
+    {
+      "epoch": 1.2716865937420503,
+      "step": 2500,
+      "total_flos": 1.2839004390911662e+19,
+      "train_loss": 2.4182849487304687,
+      "train_runtime": 5081.644,
+      "train_samples_per_second": 3.936,
+      "train_steps_per_second": 0.492
     }
   ],
   "logging_steps": 100,
-  "max_steps": 58950,
+  "max_steps": 2500,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 30,
+  "num_train_epochs": 2,
   "save_steps": 400,
   "stateful_callbacks": {
-    "EarlyStoppingCallback": {
-      "args": {
-        "early_stopping_patience": 3,
-        "early_stopping_threshold": 0.0
-      },
-      "attributes": {
-        "early_stopping_patience_counter": 3
-      }
-    },
     "TrainerControl": {
       "args": {
         "should_epoch_stop": false,
@@ -108,7 +435,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.0370481464218854e+18,
+  "total_flos": 1.2839004390911662e+19,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null