Training in progress, step 776, checkpoint

last-checkpoint/optimizer_0/__0_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e243eb5a2e392baf6fbbd34e2880b92b192e8406253f7dbfdcc5bf34a27d48d
+oid sha256:0f43f1400398e4c8505720effcc16af684c412b5638dba7f3a1aebf4a06cc201
 size 13934748

last-checkpoint/optimizer_0/__1_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:35be897abb2edd59e7e14cd584897fafab6ebcbe9d7b044ec3c0f24465e96641
+oid sha256:020d542507719a4eb2ab83beed83011776a0481d28e239449ece4a1132799518
 size 13999412

last-checkpoint/optimizer_0/__2_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:756519000cbd96a62a18f4b2593ac1531235ff43f81a97c750d5d508e0bb4895
+oid sha256:6f40acb7ba2ebf0ad18f4cab0b9f0d53f945042effd89e47fb9a8b7ccab730bd
 size 13990904

last-checkpoint/optimizer_0/__3_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9503408f5508841046560c3ab2f47fba69b29194960b3e332e210fb40344d3dc
+oid sha256:e04417cf7b8930a23dde2fa70a0f93b67ae1d9250b82485eb0de6d821acbc32d
 size 13990904

last-checkpoint/pytorch_model_fsdp_0/__0_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:526dcd0450898f0723267644199cfb9a8075bd3c8768053837a8471c3071e1f8
+oid sha256:7f3439420f05cede144ac4d240b8b0b215387dbe8143269602efa71be4361358
 size 6966784

last-checkpoint/pytorch_model_fsdp_0/__1_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b8a091054d4117d69c92fe61df180bdff12c693387e2bf9406cb7a0c62fb550
+oid sha256:944dc60c3baa584c4911758be20c6546b7d8e81f6a0cbb615f3b61e9148cf32b
 size 6966784

last-checkpoint/pytorch_model_fsdp_0/__2_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:97f4e093aada0ef569e1477e89853408435b9af062279297db8b59881fcbb194
+oid sha256:1ef3c9204dba34dd140c5701148d798c401bac1df0b7b31998ff88e7e10ae2b1
 size 6966784

last-checkpoint/pytorch_model_fsdp_0/__3_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69c3c82ad5b7f6e2fcf2155f310ae31260fa6abcafc63890af31474a7296f19d
+oid sha256:37e7acdf2f27b8bdd158b1883d758f2c93dd6ce84922dd99b3cdac6a5117b556
 size 6966784

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6a53af4113d719a74be72b9b96e06ddfcf589f1c186482a58b06d91bc3e4d34
+oid sha256:ecbf30739d47702ff1dc62fda61a8b804166ddf0362f853a15764a0f6ca810ca
 size 15088

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e3f79573df50dcf97b1bf70e5483cf9597da1b8a05214eb274fc3c3833cf3ad5
+oid sha256:62ea2caeae1f19550ff4c866c756827368743f8de236623a1b0db1ace7686574
 size 15088

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f80c1cfb5ea86276cd48dd4a859d36d824a2fd5563d437a8a0c4be3eb6200b32
+oid sha256:c1c459f1b116355fb0f3bdc3f65b9adc53657df08c616f375c6b5ee9a3f2a5eb
 size 15088

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:714110b482b507e1fb6219bf62e8ec1001b632b03d77db23a105560f7b48af04
+oid sha256:6e129f010772342f4e564c483d7381dc6b23561fca445914ea8c159bc5d7a2ed
 size 15088

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b38c3c47d3e1cfb698a7e53778b8d216b765546387a9e6bd3f149bea8249062
+oid sha256:836d724299dd150551400174e98221fb5b90ef0e07f83661da8beb5333f31043
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9020618556701031,
+  "epoch": 1.0,
   "eval_steps": 20,
-  "global_step": 700,
+  "global_step": 776,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5339,6 +5339,574 @@
       "eval_samples_per_second": 5.311,
       "eval_steps_per_second": 0.175,
       "step": 700
+    },
+    {
+      "epoch": 0.9033505154639175,
+      "grad_norm": 3.9940011501312256,
+      "learning_rate": 5.643565211494285e-07,
+      "loss": 0.0374,
+      "step": 701
+    },
+    {
+      "epoch": 0.904639175257732,
+      "grad_norm": 0.029992813244462013,
+      "learning_rate": 5.495458148283505e-07,
+      "loss": 0.0003,
+      "step": 702
+    },
+    {
+      "epoch": 0.9059278350515464,
+      "grad_norm": 0.03322442248463631,
+      "learning_rate": 5.349265521053603e-07,
+      "loss": 0.0002,
+      "step": 703
+    },
+    {
+      "epoch": 0.9072164948453608,
+      "grad_norm": 1.478757381439209,
+      "learning_rate": 5.204990291317535e-07,
+      "loss": 0.0026,
+      "step": 704
+    },
+    {
+      "epoch": 0.9085051546391752,
+      "grad_norm": 3.334146499633789,
+      "learning_rate": 5.062635381746362e-07,
+      "loss": 0.0251,
+      "step": 705
+    },
+    {
+      "epoch": 0.9097938144329897,
+      "grad_norm": 0.7413277626037598,
+      "learning_rate": 4.92220367611006e-07,
+      "loss": 0.0037,
+      "step": 706
+    },
+    {
+      "epoch": 0.9110824742268041,
+      "grad_norm": 0.4254130423069,
+      "learning_rate": 4.783698019219118e-07,
+      "loss": 0.0014,
+      "step": 707
+    },
+    {
+      "epoch": 0.9123711340206185,
+      "grad_norm": 2.775209426879883,
+      "learning_rate": 4.647121216866857e-07,
+      "loss": 0.026,
+      "step": 708
+    },
+    {
+      "epoch": 0.913659793814433,
+      "grad_norm": 1.4021016359329224,
+      "learning_rate": 4.512476035772628e-07,
+      "loss": 0.0094,
+      "step": 709
+    },
+    {
+      "epoch": 0.9149484536082474,
+      "grad_norm": 0.19824525713920593,
+      "learning_rate": 4.3797652035257544e-07,
+      "loss": 0.0016,
+      "step": 710
+    },
+    {
+      "epoch": 0.9162371134020618,
+      "grad_norm": 1.9552204608917236,
+      "learning_rate": 4.248991408530279e-07,
+      "loss": 0.0136,
+      "step": 711
+    },
+    {
+      "epoch": 0.9175257731958762,
+      "grad_norm": 0.1556915044784546,
+      "learning_rate": 4.1201572999505e-07,
+      "loss": 0.0004,
+      "step": 712
+    },
+    {
+      "epoch": 0.9188144329896907,
+      "grad_norm": 4.923846244812012,
+      "learning_rate": 3.9932654876573164e-07,
+      "loss": 0.0145,
+      "step": 713
+    },
+    {
+      "epoch": 0.9201030927835051,
+      "grad_norm": 0.12035495787858963,
+      "learning_rate": 3.8683185421753313e-07,
+      "loss": 0.0005,
+      "step": 714
+    },
+    {
+      "epoch": 0.9213917525773195,
+      "grad_norm": 1.2554630041122437,
+      "learning_rate": 3.74531899463082e-07,
+      "loss": 0.0086,
+      "step": 715
+    },
+    {
+      "epoch": 0.9226804123711341,
+      "grad_norm": 0.042812976986169815,
+      "learning_rate": 3.6242693367004365e-07,
+      "loss": 0.0002,
+      "step": 716
+    },
+    {
+      "epoch": 0.9239690721649485,
+      "grad_norm": 0.1420414000749588,
+      "learning_rate": 3.5051720205606877e-07,
+      "loss": 0.0009,
+      "step": 717
+    },
+    {
+      "epoch": 0.9252577319587629,
+      "grad_norm": 0.12338154017925262,
+      "learning_rate": 3.38802945883836e-07,
+      "loss": 0.001,
+      "step": 718
+    },
+    {
+      "epoch": 0.9265463917525774,
+      "grad_norm": 1.5074167251586914,
+      "learning_rate": 3.2728440245615724e-07,
+      "loss": 0.004,
+      "step": 719
+    },
+    {
+      "epoch": 0.9278350515463918,
+      "grad_norm": 4.496931076049805,
+      "learning_rate": 3.1596180511117235e-07,
+      "loss": 0.0144,
+      "step": 720
+    },
+    {
+      "epoch": 0.9278350515463918,
+      "eval_accuracy": 0.9975173783515392,
+      "eval_f1": 0.9557522123893806,
+      "eval_loss": 0.013383620418608189,
+      "eval_precision": 0.9642857142857143,
+      "eval_recall": 0.9473684210526315,
+      "eval_runtime": 85.8622,
+      "eval_samples_per_second": 5.299,
+      "eval_steps_per_second": 0.175,
+      "step": 720
+    },
+    {
+      "epoch": 0.9291237113402062,
+      "grad_norm": 3.5359578132629395,
+      "learning_rate": 3.048353832176221e-07,
+      "loss": 0.0237,
+      "step": 721
+    },
+    {
+      "epoch": 0.9304123711340206,
+      "grad_norm": 0.27863824367523193,
+      "learning_rate": 2.939053621702015e-07,
+      "loss": 0.0012,
+      "step": 722
+    },
+    {
+      "epoch": 0.9317010309278351,
+      "grad_norm": 2.338470935821533,
+      "learning_rate": 2.83171963384995e-07,
+      "loss": 0.0099,
+      "step": 723
+    },
+    {
+      "epoch": 0.9329896907216495,
+      "grad_norm": 4.0552592277526855,
+      "learning_rate": 2.7263540429498747e-07,
+      "loss": 0.0294,
+      "step": 724
+    },
+    {
+      "epoch": 0.9342783505154639,
+      "grad_norm": 1.4125486612319946,
+      "learning_rate": 2.6229589834566807e-07,
+      "loss": 0.0324,
+      "step": 725
+    },
+    {
+      "epoch": 0.9355670103092784,
+      "grad_norm": 3.3217031955718994,
+      "learning_rate": 2.5215365499069446e-07,
+      "loss": 0.0062,
+      "step": 726
+    },
+    {
+      "epoch": 0.9368556701030928,
+      "grad_norm": 0.8493993878364563,
+      "learning_rate": 2.4220887968765873e-07,
+      "loss": 0.002,
+      "step": 727
+    },
+    {
+      "epoch": 0.9381443298969072,
+      "grad_norm": 3.687810182571411,
+      "learning_rate": 2.3246177389392388e-07,
+      "loss": 0.037,
+      "step": 728
+    },
+    {
+      "epoch": 0.9394329896907216,
+      "grad_norm": 0.4572630822658539,
+      "learning_rate": 2.229125350625394e-07,
+      "loss": 0.0012,
+      "step": 729
+    },
+    {
+      "epoch": 0.9407216494845361,
+      "grad_norm": 0.3448236882686615,
+      "learning_rate": 2.1356135663824328e-07,
+      "loss": 0.0017,
+      "step": 730
+    },
+    {
+      "epoch": 0.9420103092783505,
+      "grad_norm": 1.1979801654815674,
+      "learning_rate": 2.0440842805354522e-07,
+      "loss": 0.0174,
+      "step": 731
+    },
+    {
+      "epoch": 0.9432989690721649,
+      "grad_norm": 0.3525691330432892,
+      "learning_rate": 1.9545393472488738e-07,
+      "loss": 0.0019,
+      "step": 732
+    },
+    {
+      "epoch": 0.9445876288659794,
+      "grad_norm": 1.4202477931976318,
+      "learning_rate": 1.866980580488842e-07,
+      "loss": 0.0269,
+      "step": 733
+    },
+    {
+      "epoch": 0.9458762886597938,
+      "grad_norm": 1.2961419820785522,
+      "learning_rate": 1.7814097539865626e-07,
+      "loss": 0.0023,
+      "step": 734
+    },
+    {
+      "epoch": 0.9471649484536082,
+      "grad_norm": 0.17165932059288025,
+      "learning_rate": 1.6978286012023225e-07,
+      "loss": 0.0009,
+      "step": 735
+    },
+    {
+      "epoch": 0.9484536082474226,
+      "grad_norm": 0.12149068713188171,
+      "learning_rate": 1.6162388152903498e-07,
+      "loss": 0.0005,
+      "step": 736
+    },
+    {
+      "epoch": 0.9497422680412371,
+      "grad_norm": 0.8597332835197449,
+      "learning_rate": 1.5366420490645738e-07,
+      "loss": 0.0028,
+      "step": 737
+    },
+    {
+      "epoch": 0.9510309278350515,
+      "grad_norm": 0.13404878973960876,
+      "learning_rate": 1.4590399149650769e-07,
+      "loss": 0.0004,
+      "step": 738
+    },
+    {
+      "epoch": 0.9523195876288659,
+      "grad_norm": 1.938827633857727,
+      "learning_rate": 1.3834339850254952e-07,
+      "loss": 0.0083,
+      "step": 739
+    },
+    {
+      "epoch": 0.9536082474226805,
+      "grad_norm": 1.4186664819717407,
+      "learning_rate": 1.309825790841146e-07,
+      "loss": 0.0284,
+      "step": 740
+    },
+    {
+      "epoch": 0.9536082474226805,
+      "eval_accuracy": 0.9980139026812314,
+      "eval_f1": 0.9642857142857143,
+      "eval_loss": 0.013383138924837112,
+      "eval_precision": 0.9818181818181818,
+      "eval_recall": 0.9473684210526315,
+      "eval_runtime": 85.8161,
+      "eval_samples_per_second": 5.302,
+      "eval_steps_per_second": 0.175,
+      "step": 740
+    },
+    {
+      "epoch": 0.9548969072164949,
+      "grad_norm": 1.9752976894378662,
+      "learning_rate": 1.2382168235379742e-07,
+      "loss": 0.0028,
+      "step": 741
+    },
+    {
+      "epoch": 0.9561855670103093,
+      "grad_norm": 4.70041036605835,
+      "learning_rate": 1.1686085337423991e-07,
+      "loss": 0.0102,
+      "step": 742
+    },
+    {
+      "epoch": 0.9574742268041238,
+      "grad_norm": 3.6995646953582764,
+      "learning_rate": 1.1010023315518592e-07,
+      "loss": 0.0218,
+      "step": 743
+    },
+    {
+      "epoch": 0.9587628865979382,
+      "grad_norm": 2.3631069660186768,
+      "learning_rate": 1.0353995865063138e-07,
+      "loss": 0.0306,
+      "step": 744
+    },
+    {
+      "epoch": 0.9600515463917526,
+      "grad_norm": 0.07328186929225922,
+      "learning_rate": 9.718016275604759e-08,
+      "loss": 0.0004,
+      "step": 745
+    },
+    {
+      "epoch": 0.961340206185567,
+      "grad_norm": 0.09281091392040253,
+      "learning_rate": 9.10209743056889e-08,
+      "loss": 0.0006,
+      "step": 746
+    },
+    {
+      "epoch": 0.9626288659793815,
+      "grad_norm": 2.129312753677368,
+      "learning_rate": 8.506251806997934e-08,
+      "loss": 0.0418,
+      "step": 747
+    },
+    {
+      "epoch": 0.9639175257731959,
+      "grad_norm": 0.45759478211402893,
+      "learning_rate": 7.930491475299229e-08,
+      "loss": 0.001,
+      "step": 748
+    },
+    {
+      "epoch": 0.9652061855670103,
+      "grad_norm": 0.9310470819473267,
+      "learning_rate": 7.37482809900003e-08,
+      "loss": 0.0023,
+      "step": 749
+    },
+    {
+      "epoch": 0.9664948453608248,
+      "grad_norm": 0.8306396007537842,
+      "learning_rate": 6.839272934511143e-08,
+      "loss": 0.0038,
+      "step": 750
+    },
+    {
+      "epoch": 0.9677835051546392,
+      "grad_norm": 0.3015538156032562,
+      "learning_rate": 6.323836830899321e-08,
+      "loss": 0.001,
+      "step": 751
+    },
+    {
+      "epoch": 0.9690721649484536,
+      "grad_norm": 2.0582327842712402,
+      "learning_rate": 5.828530229667228e-08,
+      "loss": 0.0127,
+      "step": 752
+    },
+    {
+      "epoch": 0.970360824742268,
+      "grad_norm": 0.13917666673660278,
+      "learning_rate": 5.353363164541825e-08,
+      "loss": 0.001,
+      "step": 753
+    },
+    {
+      "epoch": 0.9716494845360825,
+      "grad_norm": 1.766170620918274,
+      "learning_rate": 4.898345261271531e-08,
+      "loss": 0.0033,
+      "step": 754
+    },
+    {
+      "epoch": 0.9729381443298969,
+      "grad_norm": 2.295456647872925,
+      "learning_rate": 4.463485737430606e-08,
+      "loss": 0.0045,
+      "step": 755
+    },
+    {
+      "epoch": 0.9742268041237113,
+      "grad_norm": 3.3458142280578613,
+      "learning_rate": 4.0487934022328533e-08,
+      "loss": 0.0081,
+      "step": 756
+    },
+    {
+      "epoch": 0.9755154639175257,
+      "grad_norm": 3.1655807495117188,
+      "learning_rate": 3.654276656353206e-08,
+      "loss": 0.0101,
+      "step": 757
+    },
+    {
+      "epoch": 0.9768041237113402,
+      "grad_norm": 0.16411763429641724,
+      "learning_rate": 3.27994349175742e-08,
+      "loss": 0.0005,
+      "step": 758
+    },
+    {
+      "epoch": 0.9780927835051546,
+      "grad_norm": 3.4274473190307617,
+      "learning_rate": 2.9258014915399813e-08,
+      "loss": 0.0037,
+      "step": 759
+    },
+    {
+      "epoch": 0.979381443298969,
+      "grad_norm": 1.7726308107376099,
+      "learning_rate": 2.591857829770672e-08,
+      "loss": 0.0066,
+      "step": 760
+    },
+    {
+      "epoch": 0.979381443298969,
+      "eval_accuracy": 0.9975173783515392,
+      "eval_f1": 0.9557522123893806,
+      "eval_loss": 0.013445839285850525,
+      "eval_precision": 0.9642857142857143,
+      "eval_recall": 0.9473684210526315,
+      "eval_runtime": 86.1309,
+      "eval_samples_per_second": 5.283,
+      "eval_steps_per_second": 0.174,
+      "step": 760
+    },
+    {
+      "epoch": 0.9806701030927835,
+      "grad_norm": 3.986452579498291,
+      "learning_rate": 2.278119271349466e-08,
+      "loss": 0.0265,
+      "step": 761
+    },
+    {
+      "epoch": 0.9819587628865979,
+      "grad_norm": 1.644910454750061,
+      "learning_rate": 1.984592171869082e-08,
+      "loss": 0.0039,
+      "step": 762
+    },
+    {
+      "epoch": 0.9832474226804123,
+      "grad_norm": 0.0761791542172432,
+      "learning_rate": 1.711282477486642e-08,
+      "loss": 0.0003,
+      "step": 763
+    },
+    {
+      "epoch": 0.9845360824742269,
+      "grad_norm": 0.8887882828712463,
+      "learning_rate": 1.4581957248026579e-08,
+      "loss": 0.0047,
+      "step": 764
+    },
+    {
+      "epoch": 0.9858247422680413,
+      "grad_norm": 1.5930033922195435,
+      "learning_rate": 1.2253370407495636e-08,
+      "loss": 0.0062,
+      "step": 765
+    },
+    {
+      "epoch": 0.9871134020618557,
+      "grad_norm": 0.7668823599815369,
+      "learning_rate": 1.0127111424872437e-08,
+      "loss": 0.0027,
+      "step": 766
+    },
+    {
+      "epoch": 0.9884020618556701,
+      "grad_norm": 3.0065221786499023,
+      "learning_rate": 8.203223373078883e-09,
+      "loss": 0.0152,
+      "step": 767
+    },
+    {
+      "epoch": 0.9896907216494846,
+      "grad_norm": 0.5820819735527039,
+      "learning_rate": 6.481745225485059e-09,
+      "loss": 0.0016,
+      "step": 768
+    },
+    {
+      "epoch": 0.990979381443299,
+      "grad_norm": 0.7048105597496033,
+      "learning_rate": 4.962711855120983e-09,
+      "loss": 0.0027,
+      "step": 769
+    },
+    {
+      "epoch": 0.9922680412371134,
+      "grad_norm": 2.509854555130005,
+      "learning_rate": 3.6461540339682855e-09,
+      "loss": 0.0188,
+      "step": 770
+    },
+    {
+      "epoch": 0.9935567010309279,
+      "grad_norm": 0.2154129147529602,
+      "learning_rate": 2.532098432341812e-09,
+      "loss": 0.0008,
+      "step": 771
+    },
+    {
+      "epoch": 0.9948453608247423,
+      "grad_norm": 2.393842935562134,
+      "learning_rate": 1.6205676183411733e-09,
+      "loss": 0.0079,
+      "step": 772
+    },
+    {
+      "epoch": 0.9961340206185567,
+      "grad_norm": 2.176377058029175,
+      "learning_rate": 9.115800574022171e-10,
+      "loss": 0.013,
+      "step": 773
+    },
+    {
+      "epoch": 0.9974226804123711,
+      "grad_norm": 2.1431355476379395,
+      "learning_rate": 4.0515011191621933e-10,
+      "loss": 0.0134,
+      "step": 774
+    },
+    {
+      "epoch": 0.9987113402061856,
+      "grad_norm": 1.2513763904571533,
+      "learning_rate": 1.0128804094233779e-10,
+      "loss": 0.0034,
+      "step": 775
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 2.7385754585266113,
+      "learning_rate": 0.0,
+      "loss": 0.0062,
+      "step": 776
     }
   ],
   "logging_steps": 1,
@@ -5353,12 +5921,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 2.3348483995572634e+17,
+  "total_flos": 2.5857289592242176e+17,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null