Training in progress, step 100, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:90b727176a91679f211e2860c969bb68ca60a4372d0d383f1c88ec7f7bcfb4a2
+oid sha256:657bf78fae67ab7f060e521c89071af033d2ccaafb65ee8063a6a309b5d28838
 size 8668296

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:979d06dca9d9cbdcc97fdd078826fcb882cff9033da27c6f319d5df9caff1d34
+oid sha256:bd88998b6ccade4db28e1d1fc48c4cbcfdb5f60c6ea942d1f805e4d77dac2e88
 size 17405562

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d72f2a155c1e6705b29ac673616d520187cbe56431b6f98d143b1a04b0f8a1c
+oid sha256:ecf02e2a13b83e9af6aadaf08c5e361bbd5737cddb097d92e6b793966187f367
 size 13990

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:80a908267acfc52770c5c390f56dfc4588f39112fb26e5c0d643f32d0fa785ac
+oid sha256:4092f03770c9433b4ad0ca7df919cc90154d5482e807a1d8887f8b0bb6ea9041
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 5.905511811023622,
   "best_model_checkpoint": "./exp/wft-test-model/checkpoint-80",
-  "epoch": 4.14,
+  "epoch": 5.05,
   "eval_steps": 10,
-  "global_step": 90,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -762,6 +762,7 @@
     },
     {
       "epoch": 4.05,
+      "eval_pred": null,
       "grad_norm": 0.6178313493728638,
       "learning_rate": 0.00019,
       "loss": 0.1269,
@@ -769,6 +770,7 @@
     },
     {
       "epoch": 4.06,
+      "eval_pred": null,
       "grad_norm": 1.104316234588623,
       "learning_rate": 0.00017999999999999998,
       "loss": 0.2355,
@@ -776,6 +778,7 @@
     },
     {
       "epoch": 4.07,
+      "eval_pred": null,
       "grad_norm": 1.088135004043579,
       "learning_rate": 0.00017,
       "loss": 0.1947,
@@ -783,6 +786,7 @@
     },
     {
       "epoch": 4.08,
+      "eval_pred": null,
       "grad_norm": 0.6806421279907227,
       "learning_rate": 0.00016,
       "loss": 0.0923,
@@ -790,6 +794,7 @@
     },
     {
       "epoch": 4.09,
+      "eval_pred": null,
       "grad_norm": 0.6717160940170288,
       "learning_rate": 0.00015,
       "loss": 0.148,
@@ -797,6 +802,7 @@
     },
     {
       "epoch": 4.1,
+      "eval_pred": null,
       "grad_norm": 0.7663130760192871,
       "learning_rate": 0.00014000000000000001,
       "loss": 0.1192,
@@ -804,6 +810,7 @@
     },
     {
       "epoch": 4.11,
+      "eval_pred": null,
       "grad_norm": 1.271284818649292,
       "learning_rate": 0.00013000000000000002,
       "loss": 0.0887,
@@ -811,6 +818,7 @@
     },
     {
       "epoch": 4.12,
+      "eval_pred": null,
       "grad_norm": 0.6764338612556458,
       "learning_rate": 0.00012,
       "loss": 0.0812,
@@ -818,6 +826,7 @@
     },
     {
       "epoch": 4.13,
+      "eval_pred": null,
       "grad_norm": 0.9299374222755432,
       "learning_rate": 0.00011,
       "loss": 0.1574,
@@ -825,6 +834,7 @@
     },
     {
       "epoch": 4.14,
+      "eval_pred": null,
       "grad_norm": 0.9850154519081116,
       "learning_rate": 0.0001,
       "loss": 0.0678,
@@ -843,6 +853,90 @@
       "eval_wer": 7.086614173228346,
       "eval_wer_time": 0.007925033569335938,
       "step": 90
+    },
+    {
+      "epoch": 4.15,
+      "grad_norm": 1.0147942304611206,
+      "learning_rate": 8.999999999999999e-05,
+      "loss": 0.0695,
+      "step": 91
+    },
+    {
+      "epoch": 4.16,
+      "grad_norm": 1.2335087060928345,
+      "learning_rate": 8e-05,
+      "loss": 0.1366,
+      "step": 92
+    },
+    {
+      "epoch": 4.17,
+      "grad_norm": 0.929300844669342,
+      "learning_rate": 7.000000000000001e-05,
+      "loss": 0.0959,
+      "step": 93
+    },
+    {
+      "epoch": 4.18,
+      "grad_norm": 0.5210400223731995,
+      "learning_rate": 6e-05,
+      "loss": 0.0852,
+      "step": 94
+    },
+    {
+      "epoch": 4.19,
+      "grad_norm": 0.8599880933761597,
+      "learning_rate": 5e-05,
+      "loss": 0.0508,
+      "step": 95
+    },
+    {
+      "epoch": 5.01,
+      "grad_norm": 0.38409674167633057,
+      "learning_rate": 4e-05,
+      "loss": 0.0801,
+      "step": 96
+    },
+    {
+      "epoch": 5.02,
+      "grad_norm": 0.5891416072845459,
+      "learning_rate": 3e-05,
+      "loss": 0.0762,
+      "step": 97
+    },
+    {
+      "epoch": 5.03,
+      "grad_norm": 0.658381998538971,
+      "learning_rate": 2e-05,
+      "loss": 0.0651,
+      "step": 98
+    },
+    {
+      "epoch": 5.04,
+      "grad_norm": 0.5465841293334961,
+      "learning_rate": 1e-05,
+      "loss": 0.1254,
+      "step": 99
+    },
+    {
+      "epoch": 5.05,
+      "grad_norm": 0.4337480664253235,
+      "learning_rate": 0.0,
+      "loss": 0.078,
+      "step": 100
+    },
+    {
+      "epoch": 5.05,
+      "eval_cer": 83.23863636363636,
+      "eval_cer_time": 0.003019094467163086,
+      "eval_decode_time": 0.5298569202423096,
+      "eval_loss": 0.118515744805336,
+      "eval_pred": "| i | Label | Prediction |\n| --- | --- | --- |\n| 0 | MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL | MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPELLLTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH-\"-\"-\"TH-\"-\"-\"-\"TH-\"-\"-\"-\"TH\"-TH\"-\"-\"-\"-THTH\"-\"-TH\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-=\"-\"-\"-\"-\"-\"-\"-\"-\"-\"- |\n| 1 | NOR IS MISTER QUILTER'S MANNER LESS INTERESTING THAN HIS MATTER | NOR IS MISTER QUILTER'S MANNER LESS INTERESTING THAN HIS MATTERTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH-\"-\"\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"- |\n| 2 | HE TELLS US THAT AT THIS FESTIVE SEASON OF THE YEAR WITH CHRISTMAS AND ROAST BEEF LOOMING BEFORE US SIMILES DRAWN FROM EATING AND ITS RESULTS OCCUR MOST READILY TO THE MIND | HE TELLS US THAT AT THIS FESTIVE SEASON OF THE YEAR WITH CHRISTMAS AND ROAST BEEF LOOMING BEFORE US SIMILES DRAWN FROM EATING AND ITS RESULTS OCCUR MOST READILY TO THE MINDTHTHTHTHTHTHTHTHTHTHTH%THTHTHTHTHTHTHTHTHTHTHTHTHTH%THTH%%%%%%%%THTH%%THTHTH%%%%%%%%%%%%%%%%%%%%%%%% |\n| 3 | HE HAS GRAVE DOUBTS WHETHER SIR FREDERICK LEIGHTON'S WORK IS REALLY GREEK AFTER ALL AND CAN DISCOVER IN IT BUT LITTLE OF ROCKY ITHACA | HE HAS GRAVE DOBTS WHETHER SIR FREDERICK LEIGHTON'S WORK IS REALLY GREEK AFTER ALL AND CAN DISCOVER IN IT BUT LITTLE OF ROCKY ITHACATHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH%%TH%% |\n| 4 | LINNELL'S PICTURES ARE A SORT OF UP GUARDS AND AT EM PAINTINGS AND MASON'S EXQUISITE IDYLLS ARE AS NATIONAL AS A JINGO POEM MISTER BIRKET FOSTER'S LANDSCAPES SMILE AT ONE MUCH IN THE SAME WAY THAT MISTER CARKER USED TO FLASH HIS TEETH AND MISTER JOHN COLLIER GIVES HIS SITTER A CHEERFUL SLAP ON THE BACK BEFORE HE SAYS LIKE A SHAMPOOER IN A TURKISH BATH NEXT MAN | LINNELL'S PICTURES ARE A SORT OF UP GUARDS AND AT EM PAINTINGS AND MASON'S EXQUISITE IDIDLLS ARE AS NATIONAL AS A JINGO POEM MISTER BIRKET FOSTER'S LANDSCAPES SMILE AT ONE MUCH IN THE SAME WAY THAT MISTER CARKER USED TO FLASH HIS TEETH AND MISTER JOHN COLLIER GIVES HIS SITTER A CHEERFUL SLAP ON THE BACK BEFORE HE SAYS LIKE A SHAMPOOER IN A TURKISH BATH NEXT MAN |\n| 5 | IT IS OBVIOUSLY UNNECESSARY FOR US TO POINT OUT HOW LUMINOUS THESE CRITICISMS ARE HOW DELICATE IN EXPRESSION | IT IS OBVIOUSLY UNNECESSARY FOR US TO POINT OUT HOW LUMINOUS THESE CRITICISMS ARE HOW DELICATE IN EXPRESSIONTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\" |\n| 6 | ON THE GENERAL PRINCIPLES OF ART MISTER QUILTER WRITES WITH EQUAL LUCIDITY | ON THE GENERAL PRINCIPLES OF ART MISTER QUILTER WRITES WITH EQUAL LUCIDITYTHTHTHTHTHTHTH \u266a \u266aTH \u266a%%==\"-\"-===\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"- |\n| 7 | PAINTING HE TELLS US IS OF A DIFFERENT QUALITY TO MATHEMATICS AND FINISH IN ART IS ADDING MORE FACT | PAINTING HE TELLS US IS OF A DIFFERENT QUALITY TO MATHEMATICS AND FINISH IN ART IS ADDING MORE FACT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\"-\"-%%%%%%%\"-\"-%%\"-%%%% |\n| 8 | AS FOR ETCHINGS THEY ARE OF TWO KINDS BRITISH AND FOREIGN | AS FOR ETCHINGS THEY ARE OF TWO KINDS BRITISH AND FOREIGNTHTHTHTHTHTHTHTHTHTH-\"-\"-\"-\"-\"-\"-\"-\" |\n| 9 | HE LAMENTS MOST BITTERLY THE DIVORCE THAT HAS BEEN MADE BETWEEN DECORATIVE ART AND WHAT WE USUALLY CALL PICTURES MAKES THE CUSTOMARY APPEAL TO THE LAST JUDGMENT AND REMINDS US THAT IN THE GREAT DAYS OF ART MICHAEL ANGELO WAS THE FURNISHING UPHOLSTERER | HE LAMENTS MOST BITTERLY THE DIVORCE THAT HAS BEEN MADE BETWEEN DECORATIVE ART AND WHAT WE USUALLY CALL PICTURES MAKES THE CUSTOMARY APPEAL TO THE LAST judGMENT AND REMINDS US THAT IN THE GREAT DAYS OF ART MIC ANGELO WAS THE FURNISHING UPHOLSTERER |\n",
+      "eval_runtime": 1.3526,
+      "eval_samples_per_second": 7.393,
+      "eval_steps_per_second": 1.479,
+      "eval_wer": 5.905511811023622,
+      "eval_wer_time": 0.004703044891357422,
+      "step": 100
     }
   ],
   "logging_steps": 1,
@@ -857,12 +951,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 9651138232320000.0,
+  "total_flos": 1.06772649984e+16,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null