Training in progress, step 6777, checkpoint

last-checkpoint/optimizer_0/__0_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:965566f8b9a741a6f2801dc78e4fbc5ac70240c8d6d7b5570ba0182bcd9674e9
+oid sha256:7e2e9047c1b951991a3cb533b5422cf65b7fdd99fce52ba9dad6cd543430b657
 size 13934748

last-checkpoint/optimizer_0/__1_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7212ed89906b0804da8eba1f5c500d042a2a31b594b63c7afc77b7fca62b4f05
+oid sha256:65562a7ad6cabc3a9a834233cb6b2a6418ba0f2a4995c24942bae3fcdd716740
 size 13999412

last-checkpoint/optimizer_0/__2_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93ad7abb665289229475a0dc55018b7ca6c10b70ef45f15c0b9b8f137cc5c291
+oid sha256:d1095a5622f7ae7057612f380cf391d514c18a62ac77c825675a6caf6ee67c65
 size 13990904

last-checkpoint/optimizer_0/__3_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a28d9e945552a66feca51fc9780b294ee621de58c9db83d3aefe7462105d0d49
+oid sha256:0991fb538d2fcdd7133b88b57d7408634c5853d2bea2f8795c0d2a2c04be8c2a
 size 13990904

last-checkpoint/pytorch_model_fsdp_0/__0_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49977e9fb46265ba81ad5ce120a7b938b5fafa454d7bb632a57a63f975e9f54a
+oid sha256:68623acd4c5451515db193d2aa04b8145b1f1f36417e52c36086f855dd8b168d
 size 6966784

last-checkpoint/pytorch_model_fsdp_0/__1_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a633db66552fb787cb1151b9a3e2e30b0293e84603ef7d545351fc947c5f219
+oid sha256:e32076e376b49ea017c893743edd08119946ffa77fa889406c367ad701334a0e
 size 6966784

last-checkpoint/pytorch_model_fsdp_0/__2_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f2168060d5d243a5dda1e0bc7482749ed6c7fc4cb39ff029c8a95d29643dcf6
+oid sha256:be31033ff0655091be7d5cf4fb0b2133466588c00cd4b8a9f31a6082a8afcee4
 size 6966784

last-checkpoint/pytorch_model_fsdp_0/__3_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:581e13951882957811a470d66e41e45bbc9bb66544ca2d6e3568683cc9866887
+oid sha256:6ef8509d1cfc7f1a4958331fec1ee3edaa3e6165fe80cc849e478a187e2339e3
 size 6966784

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:091b7cd663405f9e474cf640b71ae20df31b45b8cceb2d74232e5c4232ae67f5
+oid sha256:712216dc073e1dfa28326996a976c217850297a16301f8f5ccfa922b1a5dc3ee
 size 14960

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:951da6c961efeea8abd4771cf6f335146152fd6e811aedd9376cfbaf0b5c2661
+oid sha256:9028ecc7ecc1acc6c8525827a2279969075801179a1f74a70512f85eeab8816a
 size 14960

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48de50c66a37a5de2f7b1873acf38375f58754d859c5eb82d7fe707070cddd0c
+oid sha256:f69e552de9eb6a6f61b8231a8ce384288d7ac46d1fed91935b9e5cbb21762087
 size 14960

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2136ce865bd31b51bdee33783218e662d324835501f13ef2cf89d65f472e3f07
+oid sha256:633aeca10962285051f888adc2c2c409b0bf3999362ccda037a788110e685916
 size 14960

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f3d454f05c0bda87b3125802c8738baab69763f7e63757668c9f80a78618863
+oid sha256:2a317dad97e27324d1b8604ef54c8e6a4cccd5fd5efea166b6eda97fc4edd76b
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9885651051272594,
+  "epoch": 0.999926226484692,
   "eval_steps": 20,
-  "global_step": 6700,
+  "global_step": 6777,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -50939,6 +50939,581 @@
       "eval_samples_per_second": 5.902,
       "eval_steps_per_second": 0.203,
       "step": 6700
+    },
+    {
+      "epoch": 0.9887126521578753,
+      "grad_norm": 0.30346840620040894,
+      "learning_rate": 7.661683413868748e-09,
+      "loss": 0.0029,
+      "step": 6701
+    },
+    {
+      "epoch": 0.9888601991884913,
+      "grad_norm": 3.611311674118042,
+      "learning_rate": 7.461411546916264e-09,
+      "loss": 0.0608,
+      "step": 6702
+    },
+    {
+      "epoch": 0.9890077462191074,
+      "grad_norm": 3.273043632507324,
+      "learning_rate": 7.263790976492769e-09,
+      "loss": 0.0737,
+      "step": 6703
+    },
+    {
+      "epoch": 0.9891552932497234,
+      "grad_norm": 1.5908385515213013,
+      "learning_rate": 7.068821755030763e-09,
+      "loss": 0.0437,
+      "step": 6704
+    },
+    {
+      "epoch": 0.9893028402803393,
+      "grad_norm": 2.5678610801696777,
+      "learning_rate": 6.876503934262202e-09,
+      "loss": 0.1046,
+      "step": 6705
+    },
+    {
+      "epoch": 0.9894503873109554,
+      "grad_norm": 1.9680798053741455,
+      "learning_rate": 6.686837565215154e-09,
+      "loss": 0.0491,
+      "step": 6706
+    },
+    {
+      "epoch": 0.9895979343415714,
+      "grad_norm": 3.603398084640503,
+      "learning_rate": 6.499822698210478e-09,
+      "loss": 0.0613,
+      "step": 6707
+    },
+    {
+      "epoch": 0.9897454813721874,
+      "grad_norm": 2.4477665424346924,
+      "learning_rate": 6.315459382871813e-09,
+      "loss": 0.0667,
+      "step": 6708
+    },
+    {
+      "epoch": 0.9898930284028034,
+      "grad_norm": 1.95452082157135,
+      "learning_rate": 6.1337476681122556e-09,
+      "loss": 0.0635,
+      "step": 6709
+    },
+    {
+      "epoch": 0.9900405754334194,
+      "grad_norm": 0.929094672203064,
+      "learning_rate": 5.95468760214879e-09,
+      "loss": 0.0181,
+      "step": 6710
+    },
+    {
+      "epoch": 0.9901881224640354,
+      "grad_norm": 1.7649720907211304,
+      "learning_rate": 5.77827923248786e-09,
+      "loss": 0.0337,
+      "step": 6711
+    },
+    {
+      "epoch": 0.9903356694946515,
+      "grad_norm": 2.4020333290100098,
+      "learning_rate": 5.604522605937579e-09,
+      "loss": 0.0745,
+      "step": 6712
+    },
+    {
+      "epoch": 0.9904832165252674,
+      "grad_norm": 1.8048148155212402,
+      "learning_rate": 5.433417768598847e-09,
+      "loss": 0.0588,
+      "step": 6713
+    },
+    {
+      "epoch": 0.9906307635558834,
+      "grad_norm": 1.9580947160720825,
+      "learning_rate": 5.264964765870906e-09,
+      "loss": 0.0477,
+      "step": 6714
+    },
+    {
+      "epoch": 0.9907783105864995,
+      "grad_norm": 1.246323823928833,
+      "learning_rate": 5.099163642449112e-09,
+      "loss": 0.0336,
+      "step": 6715
+    },
+    {
+      "epoch": 0.9909258576171155,
+      "grad_norm": 1.8688416481018066,
+      "learning_rate": 4.9360144423260535e-09,
+      "loss": 0.0259,
+      "step": 6716
+    },
+    {
+      "epoch": 0.9910734046477314,
+      "grad_norm": 1.5381361246109009,
+      "learning_rate": 4.775517208788217e-09,
+      "loss": 0.0298,
+      "step": 6717
+    },
+    {
+      "epoch": 0.9912209516783475,
+      "grad_norm": 1.709076166152954,
+      "learning_rate": 4.617671984420425e-09,
+      "loss": 0.0542,
+      "step": 6718
+    },
+    {
+      "epoch": 0.9913684987089635,
+      "grad_norm": 1.3890794515609741,
+      "learning_rate": 4.462478811104731e-09,
+      "loss": 0.0558,
+      "step": 6719
+    },
+    {
+      "epoch": 0.9915160457395795,
+      "grad_norm": 0.8841899633407593,
+      "learning_rate": 4.309937730015978e-09,
+      "loss": 0.0172,
+      "step": 6720
+    },
+    {
+      "epoch": 0.9915160457395795,
+      "eval_accuracy": 0.9782923299565847,
+      "eval_f1": 0.9629629629629629,
+      "eval_loss": 0.055379100143909454,
+      "eval_precision": 0.9798994974874372,
+      "eval_recall": 0.9466019417475728,
+      "eval_runtime": 50.3029,
+      "eval_samples_per_second": 5.785,
+      "eval_steps_per_second": 0.199,
+      "step": 6720
+    },
+    {
+      "epoch": 0.9916635927701954,
+      "grad_norm": 1.6927621364593506,
+      "learning_rate": 4.160048781628456e-09,
+      "loss": 0.0152,
+      "step": 6721
+    },
+    {
+      "epoch": 0.9918111398008115,
+      "grad_norm": 0.853327214717865,
+      "learning_rate": 4.012812005712574e-09,
+      "loss": 0.0098,
+      "step": 6722
+    },
+    {
+      "epoch": 0.9919586868314275,
+      "grad_norm": 3.5236921310424805,
+      "learning_rate": 3.8682274413337495e-09,
+      "loss": 0.0527,
+      "step": 6723
+    },
+    {
+      "epoch": 0.9921062338620436,
+      "grad_norm": 2.9586477279663086,
+      "learning_rate": 3.726295126853519e-09,
+      "loss": 0.0441,
+      "step": 6724
+    },
+    {
+      "epoch": 0.9922537808926596,
+      "grad_norm": 0.3246397376060486,
+      "learning_rate": 3.5870150999317567e-09,
+      "loss": 0.0029,
+      "step": 6725
+    },
+    {
+      "epoch": 0.9924013279232755,
+      "grad_norm": 1.3314762115478516,
+      "learning_rate": 3.450387397522237e-09,
+      "loss": 0.025,
+      "step": 6726
+    },
+    {
+      "epoch": 0.9925488749538915,
+      "grad_norm": 1.7466126680374146,
+      "learning_rate": 3.3164120558759614e-09,
+      "loss": 0.0336,
+      "step": 6727
+    },
+    {
+      "epoch": 0.9926964219845076,
+      "grad_norm": 0.8817312121391296,
+      "learning_rate": 3.185089110542272e-09,
+      "loss": 0.0208,
+      "step": 6728
+    },
+    {
+      "epoch": 0.9928439690151236,
+      "grad_norm": 1.9117987155914307,
+      "learning_rate": 3.056418596362187e-09,
+      "loss": 0.0431,
+      "step": 6729
+    },
+    {
+      "epoch": 0.9929915160457395,
+      "grad_norm": 5.536546230316162,
+      "learning_rate": 2.9304005474761755e-09,
+      "loss": 0.1609,
+      "step": 6730
+    },
+    {
+      "epoch": 0.9931390630763556,
+      "grad_norm": 4.430239677429199,
+      "learning_rate": 2.8070349973219336e-09,
+      "loss": 0.1037,
+      "step": 6731
+    },
+    {
+      "epoch": 0.9932866101069716,
+      "grad_norm": 1.1493936777114868,
+      "learning_rate": 2.6863219786299466e-09,
+      "loss": 0.0348,
+      "step": 6732
+    },
+    {
+      "epoch": 0.9934341571375876,
+      "grad_norm": 1.1461900472640991,
+      "learning_rate": 2.568261523430149e-09,
+      "loss": 0.0286,
+      "step": 6733
+    },
+    {
+      "epoch": 0.9935817041682036,
+      "grad_norm": 1.5313574075698853,
+      "learning_rate": 2.452853663046373e-09,
+      "loss": 0.0368,
+      "step": 6734
+    },
+    {
+      "epoch": 0.9937292511988196,
+      "grad_norm": 1.981681227684021,
+      "learning_rate": 2.3400984280985695e-09,
+      "loss": 0.0701,
+      "step": 6735
+    },
+    {
+      "epoch": 0.9938767982294356,
+      "grad_norm": 2.576758623123169,
+      "learning_rate": 2.229995848506139e-09,
+      "loss": 0.0525,
+      "step": 6736
+    },
+    {
+      "epoch": 0.9940243452600517,
+      "grad_norm": 2.1853508949279785,
+      "learning_rate": 2.1225459534801596e-09,
+      "loss": 0.0543,
+      "step": 6737
+    },
+    {
+      "epoch": 0.9941718922906676,
+      "grad_norm": 3.452421188354492,
+      "learning_rate": 2.017748771531158e-09,
+      "loss": 0.0628,
+      "step": 6738
+    },
+    {
+      "epoch": 0.9943194393212836,
+      "grad_norm": 1.7515523433685303,
+      "learning_rate": 1.915604330464671e-09,
+      "loss": 0.0685,
+      "step": 6739
+    },
+    {
+      "epoch": 0.9944669863518997,
+      "grad_norm": 2.203991413116455,
+      "learning_rate": 1.8161126573823517e-09,
+      "loss": 0.0444,
+      "step": 6740
+    },
+    {
+      "epoch": 0.9944669863518997,
+      "eval_accuracy": 0.9782923299565847,
+      "eval_f1": 0.9629629629629629,
+      "eval_loss": 0.055171407759189606,
+      "eval_precision": 0.9798994974874372,
+      "eval_recall": 0.9466019417475728,
+      "eval_runtime": 48.6957,
+      "eval_samples_per_second": 5.976,
+      "eval_steps_per_second": 0.205,
+      "step": 6740
+    },
+    {
+      "epoch": 0.9946145333825157,
+      "grad_norm": 2.4897584915161133,
+      "learning_rate": 1.7192737786819736e-09,
+      "loss": 0.0722,
+      "step": 6741
+    },
+    {
+      "epoch": 0.9947620804131316,
+      "grad_norm": 1.9944440126419067,
+      "learning_rate": 1.6250877200574278e-09,
+      "loss": 0.0995,
+      "step": 6742
+    },
+    {
+      "epoch": 0.9949096274437477,
+      "grad_norm": 2.042222738265991,
+      "learning_rate": 1.5335545064998346e-09,
+      "loss": 0.0642,
+      "step": 6743
+    },
+    {
+      "epoch": 0.9950571744743637,
+      "grad_norm": 1.745491623878479,
+      "learning_rate": 1.4446741622942128e-09,
+      "loss": 0.0506,
+      "step": 6744
+    },
+    {
+      "epoch": 0.9952047215049797,
+      "grad_norm": 0.9310182929039001,
+      "learning_rate": 1.3584467110228095e-09,
+      "loss": 0.0122,
+      "step": 6745
+    },
+    {
+      "epoch": 0.9953522685355958,
+      "grad_norm": 1.8398277759552002,
+      "learning_rate": 1.2748721755651005e-09,
+      "loss": 0.0261,
+      "step": 6746
+    },
+    {
+      "epoch": 0.9954998155662117,
+      "grad_norm": 1.677306890487671,
+      "learning_rate": 1.1939505780966809e-09,
+      "loss": 0.0358,
+      "step": 6747
+    },
+    {
+      "epoch": 0.9956473625968277,
+      "grad_norm": 1.2638682126998901,
+      "learning_rate": 1.115681940085933e-09,
+      "loss": 0.0283,
+      "step": 6748
+    },
+    {
+      "epoch": 0.9957949096274438,
+      "grad_norm": 3.8664028644561768,
+      "learning_rate": 1.040066282300689e-09,
+      "loss": 0.0514,
+      "step": 6749
+    },
+    {
+      "epoch": 0.9959424566580598,
+      "grad_norm": 1.0908302068710327,
+      "learning_rate": 9.671036248048993e-10,
+      "loss": 0.0171,
+      "step": 6750
+    },
+    {
+      "epoch": 0.9960900036886757,
+      "grad_norm": 4.517603874206543,
+      "learning_rate": 8.967939869553022e-10,
+      "loss": 0.0963,
+      "step": 6751
+    },
+    {
+      "epoch": 0.9962375507192918,
+      "grad_norm": 9.87070369720459,
+      "learning_rate": 8.291373874091957e-10,
+      "loss": 0.0511,
+      "step": 6752
+    },
+    {
+      "epoch": 0.9963850977499078,
+      "grad_norm": 1.7819442749023438,
+      "learning_rate": 7.641338441166657e-10,
+      "loss": 0.0142,
+      "step": 6753
+    },
+    {
+      "epoch": 0.9965326447805238,
+      "grad_norm": 1.9164355993270874,
+      "learning_rate": 7.017833743261371e-10,
+      "loss": 0.0333,
+      "step": 6754
+    },
+    {
+      "epoch": 0.9966801918111398,
+      "grad_norm": 1.178529143333435,
+      "learning_rate": 6.420859945788227e-10,
+      "loss": 0.0175,
+      "step": 6755
+    },
+    {
+      "epoch": 0.9968277388417558,
+      "grad_norm": 3.305959701538086,
+      "learning_rate": 5.850417207153847e-10,
+      "loss": 0.0576,
+      "step": 6756
+    },
+    {
+      "epoch": 0.9969752858723718,
+      "grad_norm": 3.1544477939605713,
+      "learning_rate": 5.306505678714934e-10,
+      "loss": 0.0754,
+      "step": 6757
+    },
+    {
+      "epoch": 0.9971228329029879,
+      "grad_norm": 1.3226299285888672,
+      "learning_rate": 4.789125504778281e-10,
+      "loss": 0.0261,
+      "step": 6758
+    },
+    {
+      "epoch": 0.9972703799336038,
+      "grad_norm": 3.1813669204711914,
+      "learning_rate": 4.2982768226229597e-10,
+      "loss": 0.1108,
+      "step": 6759
+    },
+    {
+      "epoch": 0.9974179269642198,
+      "grad_norm": 2.4142541885375977,
+      "learning_rate": 3.8339597624892366e-10,
+      "loss": 0.0481,
+      "step": 6760
+    },
+    {
+      "epoch": 0.9974179269642198,
+      "eval_accuracy": 0.9782923299565847,
+      "eval_f1": 0.9629629629629629,
+      "eval_loss": 0.05494352802634239,
+      "eval_precision": 0.9798994974874372,
+      "eval_recall": 0.9466019417475728,
+      "eval_runtime": 48.9599,
+      "eval_samples_per_second": 5.944,
+      "eval_steps_per_second": 0.204,
+      "step": 6760
+    },
+    {
+      "epoch": 0.9975654739948359,
+      "grad_norm": 1.254651665687561,
+      "learning_rate": 3.396174447556355e-10,
+      "loss": 0.0174,
+      "step": 6761
+    },
+    {
+      "epoch": 0.9977130210254519,
+      "grad_norm": 2.606041669845581,
+      "learning_rate": 2.984920994009155e-10,
+      "loss": 0.0746,
+      "step": 6762
+    },
+    {
+      "epoch": 0.9978605680560678,
+      "grad_norm": 6.167627334594727,
+      "learning_rate": 2.600199510938151e-10,
+      "loss": 0.1954,
+      "step": 6763
+    },
+    {
+      "epoch": 0.9980081150866839,
+      "grad_norm": 2.2177047729492188,
+      "learning_rate": 2.2420101004394512e-10,
+      "loss": 0.0676,
+      "step": 6764
+    },
+    {
+      "epoch": 0.9981556621172999,
+      "grad_norm": 1.7990946769714355,
+      "learning_rate": 1.9103528575370457e-10,
+      "loss": 0.0404,
+      "step": 6765
+    },
+    {
+      "epoch": 0.9983032091479159,
+      "grad_norm": 1.5802024602890015,
+      "learning_rate": 1.6052278702272107e-10,
+      "loss": 0.0352,
+      "step": 6766
+    },
+    {
+      "epoch": 0.998450756178532,
+      "grad_norm": 1.7018808126449585,
+      "learning_rate": 1.3266352194785114e-10,
+      "loss": 0.0254,
+      "step": 6767
+    },
+    {
+      "epoch": 0.9985983032091479,
+      "grad_norm": 2.102145195007324,
+      "learning_rate": 1.0745749792095971e-10,
+      "loss": 0.0427,
+      "step": 6768
+    },
+    {
+      "epoch": 0.9987458502397639,
+      "grad_norm": 2.0960288047790527,
+      "learning_rate": 8.490472163003027e-11,
+      "loss": 0.0562,
+      "step": 6769
+    },
+    {
+      "epoch": 0.99889339727038,
+      "grad_norm": 2.1226682662963867,
+      "learning_rate": 6.500519905694446e-11,
+      "loss": 0.0399,
+      "step": 6770
+    },
+    {
+      "epoch": 0.999040944300996,
+      "grad_norm": 2.805297374725342,
+      "learning_rate": 4.775893548414345e-11,
+      "loss": 0.128,
+      "step": 6771
+    },
+    {
+      "epoch": 0.9991884913316119,
+      "grad_norm": 3.2651734352111816,
+      "learning_rate": 3.316593548574609e-11,
+      "loss": 0.0469,
+      "step": 6772
+    },
+    {
+      "epoch": 0.999336038362228,
+      "grad_norm": 1.4640666246414185,
+      "learning_rate": 2.1226202935320518e-11,
+      "loss": 0.033,
+      "step": 6773
+    },
+    {
+      "epoch": 0.999483585392844,
+      "grad_norm": 1.624456524848938,
+      "learning_rate": 1.1939740999222793e-11,
+      "loss": 0.0241,
+      "step": 6774
+    },
+    {
+      "epoch": 0.99963113242346,
+      "grad_norm": 1.3273454904556274,
+      "learning_rate": 5.3065521421480356e-12,
+      "loss": 0.034,
+      "step": 6775
+    },
+    {
+      "epoch": 0.999778679454076,
+      "grad_norm": 0.9207583069801331,
+      "learning_rate": 1.3266381237997393e-12,
+      "loss": 0.026,
+      "step": 6776
+    },
+    {
+      "epoch": 0.999926226484692,
+      "grad_norm": 1.3743399381637573,
+      "learning_rate": 0.0,
+      "loss": 0.0357,
+      "step": 6777
     }
   ],
   "logging_steps": 1,
@@ -50953,12 +51528,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 2.0639708098351596e+18,
+  "total_flos": 2.0879468077357466e+18,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null