Model save

README.md

@@ -2,15 +2,9 @@
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b-dpo-full
   results: []
@@ -21,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5035
-- Rewards/chosen: -0.9681
-- Rewards/rejected: -1.9581
-- Rewards/accuracies: 0.7852
-- Rewards/margins: 0.9899
-- Logps/rejected: -458.4697
-- Logps/chosen: -359.4434
-- Logits/rejected: 1.8046
-- Logits/chosen: 0.9002
+- Loss: 0.5066
+- Rewards/chosen: -0.9998
+- Rewards/rejected: -1.9548
+- Rewards/accuracies: 0.7812
+- Rewards/margins: 0.9549
+- Logps/rejected: -458.1404
+- Logps/chosen: -362.6149
+- Logits/rejected: 2.3428
+- Logits/chosen: 1.5651
 
 ## Model description
 
@@ -68,10 +62,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5652        | 0.2092 | 100  | 0.5692          | -0.6767        | -1.2533          | 0.7070             | 0.5765          | -387.9908      | -330.3037    | -0.3959         | -0.6102       |
-| 0.5456        | 0.4184 | 200  | 0.5314          | -0.7136        | -1.4550          | 0.7656             | 0.7414          | -408.1622      | -333.9880    | 0.0197          | -0.6245       |
-| 0.494         | 0.6276 | 300  | 0.5094          | -0.8720        | -1.8005          | 0.7852             | 0.9285          | -442.7074      | -349.8286    | 1.1371          | 0.2961        |
-| 0.5019        | 0.8368 | 400  | 0.5044          | -0.9256        | -1.8971          | 0.7852             | 0.9715          | -452.3751      | -355.1891    | 1.7396          | 0.8151        |
+| 0.5644        | 0.2092 | 100  | 0.5695          | -0.6773        | -1.2063          | 0.7188             | 0.5290          | -383.2874      | -330.3586    | -0.4241         | -0.6326       |
+| 0.5445        | 0.4184 | 200  | 0.5354          | -0.6109        | -1.3936          | 0.7695             | 0.7827          | -402.0177      | -323.7153    | 0.9697          | 0.1924        |
+| 0.4909        | 0.6276 | 300  | 0.5138          | -0.9252        | -1.8338          | 0.7578             | 0.9086          | -446.0438      | -355.1510    | 2.1667          | 1.4140        |
+| 0.5036        | 0.8368 | 400  | 0.5066          | -0.9998        | -1.9548          | 0.7812             | 0.9549          | -458.1404      | -362.6149    | 2.3428          | 1.5651        |
 
 
 ### Framework versions

all_results.json

@@ -1,22 +1,9 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 0.900179386138916,
-    "eval_logits/rejected": 1.8046338558197021,
-    "eval_logps/chosen": -359.44342041015625,
-    "eval_logps/rejected": -458.4696960449219,
-    "eval_loss": 0.5034610629081726,
-    "eval_rewards/accuracies": 0.78515625,
-    "eval_rewards/chosen": -0.9681348204612732,
-    "eval_rewards/margins": 0.9899410009384155,
-    "eval_rewards/rejected": -1.9580758810043335,
-    "eval_runtime": 93.6391,
-    "eval_samples": 2000,
-    "eval_samples_per_second": 21.359,
-    "eval_steps_per_second": 0.342,
     "total_flos": 0.0,
-    "train_loss": 0.5400737299579955,
-    "train_runtime": 7416.7822,
+    "train_loss": 0.5401068107852377,
+    "train_runtime": 7660.5652,
     "train_samples": 61134,
-    "train_samples_per_second": 8.243,
-    "train_steps_per_second": 0.064
+    "train_samples_per_second": 7.98,
+    "train_steps_per_second": 0.062
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.5400737299579955,
-    "train_runtime": 7416.7822,
+    "train_loss": 0.5401068107852377,
+    "train_runtime": 7660.5652,
     "train_samples": 61134,
-    "train_samples_per_second": 8.243,
-    "train_steps_per_second": 0.064
+    "train_samples_per_second": 7.98,
+    "train_steps_per_second": 0.062
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0020920502092050207,
-      "grad_norm": 9.168965006424878,
+      "grad_norm": 9.169488248390985,
       "learning_rate": 1.0416666666666666e-08,
       "logits/chosen": -2.7655434608459473,
       "logits/rejected": -2.7158141136169434,
@@ -25,781 +25,781 @@
     },
     {
       "epoch": 0.02092050209205021,
-      "grad_norm": 9.12449411192981,
+      "grad_norm": 9.132666281035473,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.59224796295166,
-      "logits/rejected": -2.562605381011963,
-      "logps/chosen": -264.8179016113281,
-      "logps/rejected": -251.55332946777344,
+      "logits/chosen": -2.5923004150390625,
+      "logits/rejected": -2.5624728202819824,
+      "logps/chosen": -264.7728576660156,
+      "logps/rejected": -251.53372192382812,
       "loss": 0.6931,
-      "rewards/accuracies": 0.4305555522441864,
-      "rewards/chosen": -0.00024893987574614584,
-      "rewards/margins": -0.0004101950326003134,
-      "rewards/rejected": 0.00016125504043884575,
+      "rewards/accuracies": 0.3958333432674408,
+      "rewards/chosen": 0.00020155739912297577,
+      "rewards/margins": -0.00015585176879540086,
+      "rewards/rejected": 0.00035740918247029185,
       "step": 10
     },
     {
       "epoch": 0.04184100418410042,
-      "grad_norm": 8.402884792192983,
+      "grad_norm": 8.379180253402911,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.653071880340576,
-      "logits/rejected": -2.6053452491760254,
-      "logps/chosen": -281.4222106933594,
-      "logps/rejected": -296.7785949707031,
+      "logits/chosen": -2.652967929840088,
+      "logits/rejected": -2.6051251888275146,
+      "logps/chosen": -281.42034912109375,
+      "logps/rejected": -296.83856201171875,
       "loss": 0.6924,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": 0.000815415580291301,
-      "rewards/margins": 0.0005955949891358614,
-      "rewards/rejected": 0.00021982062025927007,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.000834028993267566,
+      "rewards/margins": 0.001213615294545889,
+      "rewards/rejected": -0.00037958630127832294,
       "step": 20
     },
     {
       "epoch": 0.06276150627615062,
-      "grad_norm": 8.331266596707612,
+      "grad_norm": 8.329138184590803,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.664562940597534,
-      "logits/rejected": -2.59194278717041,
-      "logps/chosen": -299.4393615722656,
-      "logps/rejected": -263.73602294921875,
+      "logits/chosen": -2.664080858230591,
+      "logits/rejected": -2.5915699005126953,
+      "logps/chosen": -299.40093994140625,
+      "logps/rejected": -263.69439697265625,
       "loss": 0.6885,
       "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.01216866448521614,
-      "rewards/margins": 0.010281884111464024,
-      "rewards/rejected": 0.0018867806065827608,
+      "rewards/chosen": 0.012552594766020775,
+      "rewards/margins": 0.010249468497931957,
+      "rewards/rejected": 0.0023031283635646105,
       "step": 30
     },
     {
       "epoch": 0.08368200836820083,
-      "grad_norm": 8.163671353448509,
+      "grad_norm": 8.173707856343666,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.556959867477417,
-      "logits/rejected": -2.5217204093933105,
-      "logps/chosen": -263.1289978027344,
-      "logps/rejected": -245.099609375,
-      "loss": 0.6777,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.039665959775447845,
-      "rewards/margins": 0.04687044769525528,
-      "rewards/rejected": -0.0072044916450977325,
+      "logits/chosen": -2.556405544281006,
+      "logits/rejected": -2.5214595794677734,
+      "logps/chosen": -263.1281433105469,
+      "logps/rejected": -245.1213836669922,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.03967439383268356,
+      "rewards/margins": 0.04709646850824356,
+      "rewards/rejected": -0.0074220686219632626,
       "step": 40
     },
     {
       "epoch": 0.10460251046025104,
-      "grad_norm": 8.415963996114856,
+      "grad_norm": 8.447504654209819,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -2.498046875,
-      "logits/rejected": -2.4585351943969727,
-      "logps/chosen": -279.64141845703125,
-      "logps/rejected": -297.4805603027344,
-      "loss": 0.6615,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0009827238973230124,
-      "rewards/margins": 0.04813830927014351,
-      "rewards/rejected": -0.04715558886528015,
+      "logits/chosen": -2.4980883598327637,
+      "logits/rejected": -2.458181381225586,
+      "logps/chosen": -279.6307678222656,
+      "logps/rejected": -297.44720458984375,
+      "loss": 0.6616,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.001089364057406783,
+      "rewards/margins": 0.04791159927845001,
+      "rewards/rejected": -0.04682223126292229,
       "step": 50
     },
     {
       "epoch": 0.12552301255230125,
-      "grad_norm": 11.37240796902324,
+      "grad_norm": 11.395386111552323,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -2.513566255569458,
-      "logits/rejected": -2.4418997764587402,
-      "logps/chosen": -273.7225036621094,
-      "logps/rejected": -312.88446044921875,
-      "loss": 0.639,
+      "logits/chosen": -2.5134081840515137,
+      "logits/rejected": -2.4417667388916016,
+      "logps/chosen": -273.7184143066406,
+      "logps/rejected": -312.87664794921875,
+      "loss": 0.6388,
       "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.020588582381606102,
-      "rewards/margins": 0.1267995536327362,
-      "rewards/rejected": -0.14738816022872925,
+      "rewards/chosen": -0.020547768101096153,
+      "rewards/margins": 0.12676255404949188,
+      "rewards/rejected": -0.14731034636497498,
       "step": 60
     },
     {
       "epoch": 0.14644351464435146,
-      "grad_norm": 10.780449834370726,
+      "grad_norm": 11.067906107691849,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -2.3635454177856445,
-      "logits/rejected": -2.3618197441101074,
-      "logps/chosen": -270.852783203125,
-      "logps/rejected": -283.3929138183594,
-      "loss": 0.6158,
+      "logits/chosen": -2.3636722564697266,
+      "logits/rejected": -2.3617496490478516,
+      "logps/chosen": -270.6546936035156,
+      "logps/rejected": -283.1260681152344,
+      "loss": 0.616,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.05183873698115349,
-      "rewards/margins": 0.21464291214942932,
-      "rewards/rejected": -0.2664816379547119,
+      "rewards/chosen": -0.04985763877630234,
+      "rewards/margins": 0.21395531296730042,
+      "rewards/rejected": -0.26381292939186096,
       "step": 70
     },
     {
       "epoch": 0.16736401673640167,
-      "grad_norm": 18.854218139571227,
+      "grad_norm": 17.68754980709847,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -2.5561511516571045,
-      "logits/rejected": -2.474567413330078,
-      "logps/chosen": -336.8047790527344,
-      "logps/rejected": -304.82586669921875,
-      "loss": 0.6023,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.14027419686317444,
-      "rewards/margins": 0.2669159471988678,
-      "rewards/rejected": -0.40719014406204224,
+      "logits/chosen": -2.552147626876831,
+      "logits/rejected": -2.4690425395965576,
+      "logps/chosen": -336.59686279296875,
+      "logps/rejected": -304.60662841796875,
+      "loss": 0.6025,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.13819542527198792,
+      "rewards/margins": 0.26680219173431396,
+      "rewards/rejected": -0.40499764680862427,
       "step": 80
     },
     {
       "epoch": 0.18828451882845187,
-      "grad_norm": 14.738392316661798,
+      "grad_norm": 15.872515076322841,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -2.082120180130005,
-      "logits/rejected": -2.03467059135437,
-      "logps/chosen": -295.71002197265625,
-      "logps/rejected": -333.998046875,
-      "loss": 0.5716,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.3656051754951477,
-      "rewards/margins": 0.4503363072872162,
-      "rewards/rejected": -0.8159415125846863,
+      "logits/chosen": -1.9810800552368164,
+      "logits/rejected": -1.9242852926254272,
+      "logps/chosen": -292.8248596191406,
+      "logps/rejected": -333.0174560546875,
+      "loss": 0.5682,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.3367532193660736,
+      "rewards/margins": 0.4693824350833893,
+      "rewards/rejected": -0.8061355352401733,
       "step": 90
     },
     {
       "epoch": 0.20920502092050208,
-      "grad_norm": 19.0889825670233,
+      "grad_norm": 17.67293052067369,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -1.0137593746185303,
-      "logits/rejected": -0.8453589677810669,
-      "logps/chosen": -314.6478271484375,
-      "logps/rejected": -370.57684326171875,
-      "loss": 0.5652,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.5427938103675842,
-      "rewards/margins": 0.5920915007591248,
-      "rewards/rejected": -1.1348851919174194,
+      "logits/chosen": -0.9724929928779602,
+      "logits/rejected": -0.756173312664032,
+      "logps/chosen": -315.0499267578125,
+      "logps/rejected": -370.2469482421875,
+      "loss": 0.5644,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.546815037727356,
+      "rewards/margins": 0.5847707986831665,
+      "rewards/rejected": -1.1315858364105225,
       "step": 100
     },
     {
       "epoch": 0.20920502092050208,
-      "eval_logits/chosen": -0.6102238297462463,
-      "eval_logits/rejected": -0.3958854675292969,
-      "eval_logps/chosen": -330.3037414550781,
-      "eval_logps/rejected": -387.99078369140625,
-      "eval_loss": 0.5692113041877747,
-      "eval_rewards/accuracies": 0.70703125,
-      "eval_rewards/chosen": -0.6767381429672241,
-      "eval_rewards/margins": 0.5765491724014282,
-      "eval_rewards/rejected": -1.2532871961593628,
-      "eval_runtime": 92.7565,
-      "eval_samples_per_second": 21.562,
-      "eval_steps_per_second": 0.345,
+      "eval_logits/chosen": -0.6325794458389282,
+      "eval_logits/rejected": -0.42406219244003296,
+      "eval_logps/chosen": -330.3586120605469,
+      "eval_logps/rejected": -383.287353515625,
+      "eval_loss": 0.5695399045944214,
+      "eval_rewards/accuracies": 0.71875,
+      "eval_rewards/chosen": -0.6772869825363159,
+      "eval_rewards/margins": 0.5289655327796936,
+      "eval_rewards/rejected": -1.2062525749206543,
+      "eval_runtime": 93.9486,
+      "eval_samples_per_second": 21.288,
+      "eval_steps_per_second": 0.341,
       "step": 100
     },
     {
       "epoch": 0.2301255230125523,
-      "grad_norm": 18.680147346638737,
+      "grad_norm": 19.504882215784345,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -0.5662704110145569,
-      "logits/rejected": -0.05761973187327385,
-      "logps/chosen": -377.0027770996094,
-      "logps/rejected": -387.5830383300781,
-      "loss": 0.5651,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.7247437238693237,
-      "rewards/margins": 0.5725878477096558,
-      "rewards/rejected": -1.297331690788269,
+      "logits/chosen": -0.6634742021560669,
+      "logits/rejected": -0.1376529037952423,
+      "logps/chosen": -370.84906005859375,
+      "logps/rejected": -380.2650146484375,
+      "loss": 0.5637,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.6632061004638672,
+      "rewards/margins": 0.5609452128410339,
+      "rewards/rejected": -1.224151372909546,
       "step": 110
     },
     {
       "epoch": 0.2510460251046025,
-      "grad_norm": 21.74920149487147,
+      "grad_norm": 22.919441722362766,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -0.38402482867240906,
-      "logits/rejected": -0.08361969143152237,
-      "logps/chosen": -353.30731201171875,
-      "logps/rejected": -394.52532958984375,
-      "loss": 0.551,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.7212796211242676,
-      "rewards/margins": 0.6250264048576355,
-      "rewards/rejected": -1.3463060855865479,
+      "logits/chosen": -0.41334447264671326,
+      "logits/rejected": -0.056368906050920486,
+      "logps/chosen": -343.59808349609375,
+      "logps/rejected": -384.07171630859375,
+      "loss": 0.5526,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.6241872906684875,
+      "rewards/margins": 0.6175822019577026,
+      "rewards/rejected": -1.241769552230835,
       "step": 120
     },
     {
       "epoch": 0.2719665271966527,
-      "grad_norm": 18.13991460993244,
+      "grad_norm": 23.51365798269679,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -0.39832931756973267,
-      "logits/rejected": -0.04852822422981262,
-      "logps/chosen": -357.71588134765625,
-      "logps/rejected": -406.4501953125,
-      "loss": 0.5346,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.7573168873786926,
-      "rewards/margins": 0.6226516962051392,
-      "rewards/rejected": -1.3799686431884766,
+      "logits/chosen": 0.1640959084033966,
+      "logits/rejected": 0.5434903502464294,
+      "logps/chosen": -387.5578308105469,
+      "logps/rejected": -431.74737548828125,
+      "loss": 0.543,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0557366609573364,
+      "rewards/margins": 0.5772033929824829,
+      "rewards/rejected": -1.6329400539398193,
       "step": 130
     },
     {
       "epoch": 0.2928870292887029,
-      "grad_norm": 19.080428474602492,
+      "grad_norm": 21.966118387144043,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -0.10298992693424225,
-      "logits/rejected": 0.23123951256275177,
-      "logps/chosen": -394.36370849609375,
-      "logps/rejected": -445.19732666015625,
-      "loss": 0.5329,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.7839752435684204,
-      "rewards/margins": 0.6615303754806519,
-      "rewards/rejected": -1.4455056190490723,
+      "logits/chosen": 0.21006186306476593,
+      "logits/rejected": 0.5628204345703125,
+      "logps/chosen": -439.6205139160156,
+      "logps/rejected": -486.50213623046875,
+      "loss": 0.547,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.2365432977676392,
+      "rewards/margins": 0.6220105886459351,
+      "rewards/rejected": -1.8585538864135742,
       "step": 140
     },
     {
       "epoch": 0.3138075313807531,
-      "grad_norm": 24.41518003453127,
+      "grad_norm": 25.216734329837617,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": 0.23929806053638458,
-      "logits/rejected": 0.663245677947998,
-      "logps/chosen": -381.1101989746094,
-      "logps/rejected": -428.31121826171875,
-      "loss": 0.5412,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.9787357449531555,
-      "rewards/margins": 0.5424004793167114,
-      "rewards/rejected": -1.5211361646652222,
+      "logits/chosen": -0.12071762979030609,
+      "logits/rejected": 0.32984182238578796,
+      "logps/chosen": -368.8601989746094,
+      "logps/rejected": -416.48101806640625,
+      "loss": 0.5444,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8562356233596802,
+      "rewards/margins": 0.5465985536575317,
+      "rewards/rejected": -1.4028340578079224,
       "step": 150
     },
     {
       "epoch": 0.33472803347280333,
-      "grad_norm": 30.00328915671681,
+      "grad_norm": 27.13158239567326,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": 0.18379352986812592,
-      "logits/rejected": 0.9613186717033386,
-      "logps/chosen": -354.5564270019531,
-      "logps/rejected": -423.7784118652344,
-      "loss": 0.5338,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.8151340484619141,
-      "rewards/margins": 0.8484436869621277,
-      "rewards/rejected": -1.6635777950286865,
+      "logits/chosen": 0.2426595389842987,
+      "logits/rejected": 0.9978505373001099,
+      "logps/chosen": -358.2662353515625,
+      "logps/rejected": -418.1039123535156,
+      "loss": 0.5383,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.8522323369979858,
+      "rewards/margins": 0.7546002864837646,
+      "rewards/rejected": -1.60683274269104,
       "step": 160
     },
     {
       "epoch": 0.35564853556485354,
-      "grad_norm": 32.90887166557451,
+      "grad_norm": 23.860913284499137,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": 0.6914592385292053,
-      "logits/rejected": 1.118165373802185,
-      "logps/chosen": -350.73297119140625,
-      "logps/rejected": -398.38079833984375,
-      "loss": 0.5397,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.014177918434143,
-      "rewards/margins": 0.6582826375961304,
-      "rewards/rejected": -1.6724605560302734,
+      "logits/chosen": 0.5063488483428955,
+      "logits/rejected": 0.9657643437385559,
+      "logps/chosen": -333.5655212402344,
+      "logps/rejected": -372.67657470703125,
+      "loss": 0.5413,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8425030708312988,
+      "rewards/margins": 0.57291579246521,
+      "rewards/rejected": -1.4154188632965088,
       "step": 170
     },
     {
       "epoch": 0.37656903765690375,
-      "grad_norm": 22.287968378600482,
+      "grad_norm": 22.58619850722941,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": 0.37205350399017334,
-      "logits/rejected": 1.014420509338379,
-      "logps/chosen": -331.5960998535156,
-      "logps/rejected": -370.4664611816406,
-      "loss": 0.5103,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.7526803612709045,
-      "rewards/margins": 0.6762296557426453,
-      "rewards/rejected": -1.4289101362228394,
+      "logits/chosen": 1.7443010807037354,
+      "logits/rejected": 2.3398690223693848,
+      "logps/chosen": -361.9225769042969,
+      "logps/rejected": -401.04473876953125,
+      "loss": 0.5088,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.0559453964233398,
+      "rewards/margins": 0.6787471771240234,
+      "rewards/rejected": -1.7346923351287842,
       "step": 180
     },
     {
       "epoch": 0.39748953974895396,
-      "grad_norm": 25.06102572422969,
+      "grad_norm": 23.3743639513868,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": 0.3608325123786926,
-      "logits/rejected": 1.0164767503738403,
-      "logps/chosen": -363.13262939453125,
-      "logps/rejected": -417.058349609375,
-      "loss": 0.5144,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.9183723330497742,
-      "rewards/margins": 0.6544338464736938,
-      "rewards/rejected": -1.5728062391281128,
+      "logits/chosen": 1.0638327598571777,
+      "logits/rejected": 1.6086857318878174,
+      "logps/chosen": -352.91046142578125,
+      "logps/rejected": -408.85784912109375,
+      "loss": 0.5196,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.8161503076553345,
+      "rewards/margins": 0.6746509671211243,
+      "rewards/rejected": -1.490801215171814,
       "step": 190
     },
     {
       "epoch": 0.41841004184100417,
-      "grad_norm": 56.1449128192664,
+      "grad_norm": 49.548116648788316,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": 0.13207800686359406,
-      "logits/rejected": 0.5271329879760742,
-      "logps/chosen": -351.56219482421875,
-      "logps/rejected": -415.3944396972656,
-      "loss": 0.5456,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.9166672825813293,
-      "rewards/margins": 0.7168152332305908,
-      "rewards/rejected": -1.6334823369979858,
+      "logits/chosen": 0.8589725494384766,
+      "logits/rejected": 1.4762446880340576,
+      "logps/chosen": -341.1542663574219,
+      "logps/rejected": -407.9635009765625,
+      "loss": 0.5445,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8125879168510437,
+      "rewards/margins": 0.7465857267379761,
+      "rewards/rejected": -1.559173583984375,
       "step": 200
     },
     {
       "epoch": 0.41841004184100417,
-      "eval_logits/chosen": -0.6244610548019409,
-      "eval_logits/rejected": 0.01971178501844406,
-      "eval_logps/chosen": -333.98797607421875,
-      "eval_logps/rejected": -408.1622314453125,
-      "eval_loss": 0.5313767194747925,
-      "eval_rewards/accuracies": 0.765625,
-      "eval_rewards/chosen": -0.7135804295539856,
-      "eval_rewards/margins": 0.7414206862449646,
-      "eval_rewards/rejected": -1.4550011157989502,
-      "eval_runtime": 92.967,
-      "eval_samples_per_second": 21.513,
-      "eval_steps_per_second": 0.344,
+      "eval_logits/chosen": 0.19241900742053986,
+      "eval_logits/rejected": 0.969687283039093,
+      "eval_logps/chosen": -323.71527099609375,
+      "eval_logps/rejected": -402.0177307128906,
+      "eval_loss": 0.5353989005088806,
+      "eval_rewards/accuracies": 0.76953125,
+      "eval_rewards/chosen": -0.6108533143997192,
+      "eval_rewards/margins": 0.7827030420303345,
+      "eval_rewards/rejected": -1.3935563564300537,
+      "eval_runtime": 93.8143,
+      "eval_samples_per_second": 21.319,
+      "eval_steps_per_second": 0.341,
       "step": 200
     },
     {
       "epoch": 0.4393305439330544,
-      "grad_norm": 28.205388358273062,
+      "grad_norm": 23.324141646005582,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -0.4052346348762512,
-      "logits/rejected": 0.4885997772216797,
-      "logps/chosen": -368.7042236328125,
-      "logps/rejected": -401.01116943359375,
-      "loss": 0.5385,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.7903989553451538,
-      "rewards/margins": 0.7071281671524048,
-      "rewards/rejected": -1.4975273609161377,
+      "logits/chosen": 0.8318442106246948,
+      "logits/rejected": 1.7365608215332031,
+      "logps/chosen": -374.9430847167969,
+      "logps/rejected": -405.85406494140625,
+      "loss": 0.5368,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8527874946594238,
+      "rewards/margins": 0.6931684613227844,
+      "rewards/rejected": -1.5459558963775635,
       "step": 210
     },
     {
       "epoch": 0.4602510460251046,
-      "grad_norm": 31.469903246953805,
+      "grad_norm": 31.425698930769954,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": 0.48687195777893066,
-      "logits/rejected": 1.3709402084350586,
-      "logps/chosen": -373.0572204589844,
-      "logps/rejected": -430.284423828125,
-      "loss": 0.5311,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.0767996311187744,
-      "rewards/margins": 0.7690645456314087,
-      "rewards/rejected": -1.8458640575408936,
+      "logits/chosen": 1.3936798572540283,
+      "logits/rejected": 2.1958301067352295,
+      "logps/chosen": -372.55242919921875,
+      "logps/rejected": -432.43658447265625,
+      "loss": 0.5238,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.071751356124878,
+      "rewards/margins": 0.7956344485282898,
+      "rewards/rejected": -1.8673858642578125,
       "step": 220
     },
     {
       "epoch": 0.4811715481171548,
-      "grad_norm": 21.508720745531846,
+      "grad_norm": 23.84771859224052,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": 0.6478261947631836,
-      "logits/rejected": 1.4287010431289673,
-      "logps/chosen": -348.21307373046875,
-      "logps/rejected": -428.8194885253906,
-      "loss": 0.5366,
+      "logits/chosen": 1.1196578741073608,
+      "logits/rejected": 1.908739686012268,
+      "logps/chosen": -338.7588806152344,
+      "logps/rejected": -430.38177490234375,
+      "loss": 0.5365,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.9925994873046875,
-      "rewards/margins": 0.8178664445877075,
-      "rewards/rejected": -1.8104660511016846,
+      "rewards/chosen": -0.8980579376220703,
+      "rewards/margins": 0.9280304908752441,
+      "rewards/rejected": -1.826088309288025,
       "step": 230
     },
     {
       "epoch": 0.502092050209205,
-      "grad_norm": 23.741045093534996,
+      "grad_norm": 25.326708869779715,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": 0.6465433835983276,
-      "logits/rejected": 1.3921277523040771,
-      "logps/chosen": -352.5728454589844,
-      "logps/rejected": -413.220947265625,
-      "loss": 0.5045,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.8883668184280396,
-      "rewards/margins": 0.7533131837844849,
-      "rewards/rejected": -1.6416800022125244,
+      "logits/chosen": 0.7311414480209351,
+      "logits/rejected": 1.439819097518921,
+      "logps/chosen": -337.96356201171875,
+      "logps/rejected": -394.6842346191406,
+      "loss": 0.5039,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.74227374792099,
+      "rewards/margins": 0.7140385508537292,
+      "rewards/rejected": -1.4563124179840088,
       "step": 240
     },
     {
       "epoch": 0.5230125523012552,
-      "grad_norm": 24.18519529429216,
+      "grad_norm": 23.27146742453404,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": 0.7936254143714905,
-      "logits/rejected": 1.7098544836044312,
-      "logps/chosen": -389.4216003417969,
-      "logps/rejected": -437.39141845703125,
-      "loss": 0.5098,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.9121864438056946,
-      "rewards/margins": 0.8662682771682739,
-      "rewards/rejected": -1.7784545421600342,
+      "logits/chosen": 1.5978848934173584,
+      "logits/rejected": 2.462660551071167,
+      "logps/chosen": -394.6571044921875,
+      "logps/rejected": -444.3811950683594,
+      "loss": 0.5066,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.9645415544509888,
+      "rewards/margins": 0.8838118314743042,
+      "rewards/rejected": -1.848353385925293,
       "step": 250
     },
     {
       "epoch": 0.5439330543933054,
-      "grad_norm": 30.73692690145265,
+      "grad_norm": 28.502041662438575,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": 0.7679815292358398,
-      "logits/rejected": 1.6959028244018555,
-      "logps/chosen": -401.0132751464844,
-      "logps/rejected": -448.0699768066406,
-      "loss": 0.509,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.0671550035476685,
-      "rewards/margins": 0.8742769956588745,
-      "rewards/rejected": -1.941432237625122,
+      "logits/chosen": 1.593186616897583,
+      "logits/rejected": 2.44622802734375,
+      "logps/chosen": -398.6466064453125,
+      "logps/rejected": -443.5439453125,
+      "loss": 0.5116,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.0434881448745728,
+      "rewards/margins": 0.8526838421821594,
+      "rewards/rejected": -1.8961719274520874,
       "step": 260
     },
     {
       "epoch": 0.5648535564853556,
-      "grad_norm": 21.173165688823637,
+      "grad_norm": 21.02294818188671,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": 0.5351714491844177,
-      "logits/rejected": 1.2392023801803589,
-      "logps/chosen": -379.1543884277344,
-      "logps/rejected": -429.35150146484375,
-      "loss": 0.5231,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.0217397212982178,
-      "rewards/margins": 0.6952935457229614,
-      "rewards/rejected": -1.7170331478118896,
+      "logits/chosen": 1.052922010421753,
+      "logits/rejected": 1.8219140768051147,
+      "logps/chosen": -361.7513122558594,
+      "logps/rejected": -412.5428771972656,
+      "loss": 0.521,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.847709059715271,
+      "rewards/margins": 0.7012378573417664,
+      "rewards/rejected": -1.5489470958709717,
       "step": 270
     },
     {
       "epoch": 0.5857740585774058,
-      "grad_norm": 28.68227240987258,
+      "grad_norm": 25.834888960693267,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": 0.8288900256156921,
-      "logits/rejected": 1.7930643558502197,
-      "logps/chosen": -372.87237548828125,
-      "logps/rejected": -432.2880859375,
-      "loss": 0.5307,
+      "logits/chosen": 1.7453187704086304,
+      "logits/rejected": 2.704537868499756,
+      "logps/chosen": -369.0428161621094,
+      "logps/rejected": -429.200927734375,
+      "loss": 0.5279,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.1114728450775146,
-      "rewards/margins": 0.7892667055130005,
-      "rewards/rejected": -1.9007394313812256,
+      "rewards/chosen": -1.073177456855774,
+      "rewards/margins": 0.7966909408569336,
+      "rewards/rejected": -1.869868278503418,
       "step": 280
     },
     {
       "epoch": 0.606694560669456,
-      "grad_norm": 24.56550896170595,
+      "grad_norm": 23.200570814556684,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": 0.7682640552520752,
-      "logits/rejected": 1.5038535594940186,
-      "logps/chosen": -370.6294860839844,
-      "logps/rejected": -434.703125,
-      "loss": 0.5206,
+      "logits/chosen": 1.8309924602508545,
+      "logits/rejected": 2.505741596221924,
+      "logps/chosen": -375.9858703613281,
+      "logps/rejected": -437.6570739746094,
+      "loss": 0.5144,
       "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.1953760385513306,
-      "rewards/margins": 0.7320847511291504,
-      "rewards/rejected": -1.92746102809906,
+      "rewards/chosen": -1.2489399909973145,
+      "rewards/margins": 0.7080605030059814,
+      "rewards/rejected": -1.957000494003296,
       "step": 290
     },
     {
       "epoch": 0.6276150627615062,
-      "grad_norm": 24.820945259716815,
+      "grad_norm": 27.865655196796528,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": 0.2875315546989441,
-      "logits/rejected": 1.408119559288025,
-      "logps/chosen": -400.6693420410156,
-      "logps/rejected": -436.7262268066406,
-      "loss": 0.494,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.9964024424552917,
-      "rewards/margins": 0.7701075077056885,
-      "rewards/rejected": -1.766510009765625,
+      "logits/chosen": 1.2169395685195923,
+      "logits/rejected": 2.430232048034668,
+      "logps/chosen": -399.8282165527344,
+      "logps/rejected": -438.07861328125,
+      "loss": 0.4909,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.987991452217102,
+      "rewards/margins": 0.7920428514480591,
+      "rewards/rejected": -1.7800343036651611,
       "step": 300
     },
     {
       "epoch": 0.6276150627615062,
-      "eval_logits/chosen": 0.29611948132514954,
-      "eval_logits/rejected": 1.1370961666107178,
-      "eval_logps/chosen": -349.82861328125,
-      "eval_logps/rejected": -442.7073669433594,
-      "eval_loss": 0.5094121098518372,
-      "eval_rewards/accuracies": 0.78515625,
-      "eval_rewards/chosen": -0.871986985206604,
-      "eval_rewards/margins": 0.9284656643867493,
-      "eval_rewards/rejected": -1.8004525899887085,
-      "eval_runtime": 93.1533,
-      "eval_samples_per_second": 21.47,
-      "eval_steps_per_second": 0.344,
+      "eval_logits/chosen": 1.4139740467071533,
+      "eval_logits/rejected": 2.1666512489318848,
+      "eval_logps/chosen": -355.1510009765625,
+      "eval_logps/rejected": -446.0438232421875,
+      "eval_loss": 0.5138201117515564,
+      "eval_rewards/accuracies": 0.7578125,
+      "eval_rewards/chosen": -0.9252106547355652,
+      "eval_rewards/margins": 0.9086061716079712,
+      "eval_rewards/rejected": -1.8338167667388916,
+      "eval_runtime": 93.7782,
+      "eval_samples_per_second": 21.327,
+      "eval_steps_per_second": 0.341,
       "step": 300
     },
     {
       "epoch": 0.6485355648535565,
-      "grad_norm": 26.748349709062442,
+      "grad_norm": 29.586629493453568,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": 0.5810573101043701,
-      "logits/rejected": 1.3016048669815063,
-      "logps/chosen": -399.5757751464844,
-      "logps/rejected": -436.1788024902344,
-      "loss": 0.4938,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.0427275896072388,
-      "rewards/margins": 0.7340670228004456,
-      "rewards/rejected": -1.776794672012329,
+      "logits/chosen": 1.761592149734497,
+      "logits/rejected": 2.4888718128204346,
+      "logps/chosen": -405.75091552734375,
+      "logps/rejected": -444.1439514160156,
+      "loss": 0.4948,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.1044785976409912,
+      "rewards/margins": 0.7519679665565491,
+      "rewards/rejected": -1.8564465045928955,
       "step": 310
     },
     {
       "epoch": 0.6694560669456067,
-      "grad_norm": 31.96747246205928,
+      "grad_norm": 28.12248662851152,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": 1.1102755069732666,
-      "logits/rejected": 2.059471368789673,
-      "logps/chosen": -419.55902099609375,
-      "logps/rejected": -446.55548095703125,
-      "loss": 0.4909,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.2584612369537354,
-      "rewards/margins": 0.7585103511810303,
-      "rewards/rejected": -2.0169715881347656,
+      "logits/chosen": 2.008812427520752,
+      "logits/rejected": 2.907062530517578,
+      "logps/chosen": -408.342529296875,
+      "logps/rejected": -436.1858825683594,
+      "loss": 0.4912,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.1462968587875366,
+      "rewards/margins": 0.7669790387153625,
+      "rewards/rejected": -1.913275957107544,
       "step": 320
     },
     {
       "epoch": 0.6903765690376569,
-      "grad_norm": 31.34877880671993,
+      "grad_norm": 29.887208796367812,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": 0.4595663547515869,
-      "logits/rejected": 1.4255298376083374,
-      "logps/chosen": -364.82562255859375,
-      "logps/rejected": -434.95831298828125,
-      "loss": 0.4957,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.025868535041809,
-      "rewards/margins": 0.7284258008003235,
-      "rewards/rejected": -1.7542946338653564,
+      "logits/chosen": 1.4809958934783936,
+      "logits/rejected": 2.334693193435669,
+      "logps/chosen": -368.56500244140625,
+      "logps/rejected": -438.9111328125,
+      "loss": 0.4951,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0632619857788086,
+      "rewards/margins": 0.7305603623390198,
+      "rewards/rejected": -1.7938222885131836,
       "step": 330
     },
     {
       "epoch": 0.7112970711297071,
-      "grad_norm": 36.534964414934784,
+      "grad_norm": 38.50152594632388,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": 0.5645917654037476,
-      "logits/rejected": 1.5419148206710815,
-      "logps/chosen": -382.41680908203125,
-      "logps/rejected": -477.4661560058594,
-      "loss": 0.5004,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.9233899116516113,
-      "rewards/margins": 1.0502269268035889,
-      "rewards/rejected": -1.9736168384552002,
+      "logits/chosen": 1.5904924869537354,
+      "logits/rejected": 2.3698315620422363,
+      "logps/chosen": -392.74053955078125,
+      "logps/rejected": -485.83966064453125,
+      "loss": 0.5032,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.0266270637512207,
+      "rewards/margins": 1.030725121498108,
+      "rewards/rejected": -2.057352066040039,
       "step": 340
     },
     {
       "epoch": 0.7322175732217573,
-      "grad_norm": 23.351076956086505,
+      "grad_norm": 25.77639486551186,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": 0.7957900762557983,
-      "logits/rejected": 1.681909203529358,
-      "logps/chosen": -360.25079345703125,
-      "logps/rejected": -430.71343994140625,
-      "loss": 0.4759,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.0204814672470093,
-      "rewards/margins": 0.8796509504318237,
-      "rewards/rejected": -1.900132417678833,
+      "logits/chosen": 1.6322052478790283,
+      "logits/rejected": 2.4968695640563965,
+      "logps/chosen": -368.20428466796875,
+      "logps/rejected": -436.6559143066406,
+      "loss": 0.4763,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1000165939331055,
+      "rewards/margins": 0.8595396876335144,
+      "rewards/rejected": -1.9595565795898438,
       "step": 350
     },
     {
       "epoch": 0.7531380753138075,
-      "grad_norm": 24.72192771367193,
+      "grad_norm": 22.519856777137164,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": 0.7120740413665771,
-      "logits/rejected": 1.9698877334594727,
-      "logps/chosen": -409.35009765625,
-      "logps/rejected": -462.37213134765625,
-      "loss": 0.5113,
+      "logits/chosen": 1.4749292135238647,
+      "logits/rejected": 2.6089377403259277,
+      "logps/chosen": -413.32354736328125,
+      "logps/rejected": -461.50213623046875,
+      "loss": 0.5153,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.066542387008667,
-      "rewards/margins": 0.944599449634552,
-      "rewards/rejected": -2.0111420154571533,
+      "rewards/chosen": -1.1062767505645752,
+      "rewards/margins": 0.8961655497550964,
+      "rewards/rejected": -2.0024421215057373,
       "step": 360
     },
     {
       "epoch": 0.7740585774058577,
-      "grad_norm": 25.85275095494242,
+      "grad_norm": 25.05710591816908,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": 1.2129358053207397,
-      "logits/rejected": 2.045804738998413,
-      "logps/chosen": -397.6884460449219,
-      "logps/rejected": -434.45635986328125,
-      "loss": 0.5097,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.2235265970230103,
-      "rewards/margins": 0.7381704449653625,
-      "rewards/rejected": -1.961696982383728,
+      "logits/chosen": 1.9852964878082275,
+      "logits/rejected": 2.7102103233337402,
+      "logps/chosen": -397.772216796875,
+      "logps/rejected": -431.29083251953125,
+      "loss": 0.5068,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.2243640422821045,
+      "rewards/margins": 0.7056778073310852,
+      "rewards/rejected": -1.930042028427124,
       "step": 370
     },
     {
       "epoch": 0.7949790794979079,
-      "grad_norm": 26.772769459890974,
+      "grad_norm": 26.679151914355828,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": 1.6128628253936768,
-      "logits/rejected": 2.2228505611419678,
-      "logps/chosen": -372.97344970703125,
-      "logps/rejected": -485.552734375,
-      "loss": 0.5079,
+      "logits/chosen": 2.23157000541687,
+      "logits/rejected": 2.7516064643859863,
+      "logps/chosen": -376.5492248535156,
+      "logps/rejected": -489.54620361328125,
+      "loss": 0.5026,
       "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.3349088430404663,
-      "rewards/margins": 0.9340818524360657,
-      "rewards/rejected": -2.2689907550811768,
+      "rewards/chosen": -1.3706672191619873,
+      "rewards/margins": 0.9382585287094116,
+      "rewards/rejected": -2.3089256286621094,
       "step": 380
     },
     {
       "epoch": 0.8158995815899581,
-      "grad_norm": 24.093673944544452,
+      "grad_norm": 24.84556158087066,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": 1.175621509552002,
-      "logits/rejected": 1.7995212078094482,
-      "logps/chosen": -421.20068359375,
-      "logps/rejected": -482.9652404785156,
-      "loss": 0.4904,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1557972431182861,
-      "rewards/margins": 0.8340117335319519,
-      "rewards/rejected": -1.9898090362548828,
+      "logits/chosen": 1.709517240524292,
+      "logits/rejected": 2.2737374305725098,
+      "logps/chosen": -423.19622802734375,
+      "logps/rejected": -487.203857421875,
+      "loss": 0.4908,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1757529973983765,
+      "rewards/margins": 0.8564424514770508,
+      "rewards/rejected": -2.0321953296661377,
       "step": 390
     },
     {
       "epoch": 0.8368200836820083,
-      "grad_norm": 25.903427781490553,
+      "grad_norm": 25.903753753002636,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": 1.2189610004425049,
-      "logits/rejected": 2.2791786193847656,
-      "logps/chosen": -381.3320007324219,
-      "logps/rejected": -468.74359130859375,
-      "loss": 0.5019,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.1078449487686157,
-      "rewards/margins": 0.9100783467292786,
-      "rewards/rejected": -2.01792311668396,
+      "logits/chosen": 1.9856996536254883,
+      "logits/rejected": 2.878208637237549,
+      "logps/chosen": -387.4896240234375,
+      "logps/rejected": -474.1192932128906,
+      "loss": 0.5036,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.1694214344024658,
+      "rewards/margins": 0.9022589921951294,
+      "rewards/rejected": -2.0716805458068848,
       "step": 400
     },
     {
       "epoch": 0.8368200836820083,
-      "eval_logits/chosen": 0.8151233792304993,
-      "eval_logits/rejected": 1.7396329641342163,
-      "eval_logps/chosen": -355.1890869140625,
-      "eval_logps/rejected": -452.37506103515625,
-      "eval_loss": 0.5043635368347168,
-      "eval_rewards/accuracies": 0.78515625,
-      "eval_rewards/chosen": -0.9255913496017456,
-      "eval_rewards/margins": 0.9715384244918823,
-      "eval_rewards/rejected": -1.8971298933029175,
-      "eval_runtime": 93.4716,
-      "eval_samples_per_second": 21.397,
-      "eval_steps_per_second": 0.342,
+      "eval_logits/chosen": 1.5651072263717651,
+      "eval_logits/rejected": 2.3428351879119873,
+      "eval_logps/chosen": -362.6148681640625,
+      "eval_logps/rejected": -458.140380859375,
+      "eval_loss": 0.5066322088241577,
+      "eval_rewards/accuracies": 0.78125,
+      "eval_rewards/chosen": -0.9998489618301392,
+      "eval_rewards/margins": 0.9549338817596436,
+      "eval_rewards/rejected": -1.9547827243804932,
+      "eval_runtime": 93.7833,
+      "eval_samples_per_second": 21.326,
+      "eval_steps_per_second": 0.341,
       "step": 400
     },
     {
       "epoch": 0.8577405857740585,
-      "grad_norm": 28.39031028613065,
+      "grad_norm": 27.223878961638523,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": 0.9069422483444214,
-      "logits/rejected": 2.2471582889556885,
-      "logps/chosen": -399.8828430175781,
-      "logps/rejected": -458.23358154296875,
-      "loss": 0.5082,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.009491205215454,
-      "rewards/margins": 0.8845922350883484,
-      "rewards/rejected": -1.8940832614898682,
+      "logits/chosen": 1.7999566793441772,
+      "logits/rejected": 2.912705183029175,
+      "logps/chosen": -406.19830322265625,
+      "logps/rejected": -464.55419921875,
+      "loss": 0.5098,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.0726454257965088,
+      "rewards/margins": 0.8846432566642761,
+      "rewards/rejected": -1.9572887420654297,
       "step": 410
     },
     {
       "epoch": 0.8786610878661087,
-      "grad_norm": 26.704848418881845,
+      "grad_norm": 24.576314036164316,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": 0.633719801902771,
-      "logits/rejected": 1.6843284368515015,
-      "logps/chosen": -395.73773193359375,
-      "logps/rejected": -453.5660095214844,
-      "loss": 0.4959,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.9866059422492981,
-      "rewards/margins": 0.8046499490737915,
-      "rewards/rejected": -1.7912559509277344,
+      "logits/chosen": 1.4636269807815552,
+      "logits/rejected": 2.3438045978546143,
+      "logps/chosen": -404.0235900878906,
+      "logps/rejected": -464.222900390625,
+      "loss": 0.4945,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.0694642066955566,
+      "rewards/margins": 0.8283600807189941,
+      "rewards/rejected": -1.8978245258331299,
       "step": 420
     },
     {
       "epoch": 0.899581589958159,
-      "grad_norm": 27.04072698273277,
+      "grad_norm": 26.751044345392213,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": 1.1087729930877686,
-      "logits/rejected": 2.018793821334839,
-      "logps/chosen": -387.2413635253906,
-      "logps/rejected": -453.595947265625,
-      "loss": 0.4749,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.0820001363754272,
-      "rewards/margins": 0.8762818574905396,
-      "rewards/rejected": -1.9582821130752563,
+      "logits/chosen": 1.9449716806411743,
+      "logits/rejected": 2.7110419273376465,
+      "logps/chosen": -393.8571472167969,
+      "logps/rejected": -460.3021545410156,
+      "loss": 0.474,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.1481584310531616,
+      "rewards/margins": 0.8771858215332031,
+      "rewards/rejected": -2.0253443717956543,
       "step": 430
     },
     {
       "epoch": 0.9205020920502092,
-      "grad_norm": 31.988161638127448,
+      "grad_norm": 30.050302790236575,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": 0.8210460543632507,
-      "logits/rejected": 1.825097680091858,
-      "logps/chosen": -378.8327331542969,
-      "logps/rejected": -445.6421813964844,
-      "loss": 0.5011,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.945022463798523,
-      "rewards/margins": 0.8203840255737305,
-      "rewards/rejected": -1.765406608581543,
+      "logits/chosen": 1.566941499710083,
+      "logits/rejected": 2.495941400527954,
+      "logps/chosen": -384.67755126953125,
+      "logps/rejected": -449.6517639160156,
+      "loss": 0.4998,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0034703016281128,
+      "rewards/margins": 0.8020319938659668,
+      "rewards/rejected": -1.8055026531219482,
       "step": 440
     },
     {
       "epoch": 0.9414225941422594,
-      "grad_norm": 24.964570138956866,
+      "grad_norm": 27.265945171563892,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": 0.8392779231071472,
-      "logits/rejected": 1.7072703838348389,
-      "logps/chosen": -397.63751220703125,
-      "logps/rejected": -497.10888671875,
-      "loss": 0.4977,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.0391228199005127,
-      "rewards/margins": 0.8608413934707642,
-      "rewards/rejected": -1.8999640941619873,
+      "logits/chosen": 1.5636577606201172,
+      "logits/rejected": 2.303377866744995,
+      "logps/chosen": -403.6795654296875,
+      "logps/rejected": -497.7889709472656,
+      "loss": 0.4996,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0995432138442993,
+      "rewards/margins": 0.8072215914726257,
+      "rewards/rejected": -1.9067647457122803,
       "step": 450
     },
     {
       "epoch": 0.9623430962343096,
-      "grad_norm": 27.14765135099825,
+      "grad_norm": 26.83929563886876,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": 0.9813226461410522,
-      "logits/rejected": 1.978114366531372,
-      "logps/chosen": -371.80206298828125,
-      "logps/rejected": -422.38787841796875,
-      "loss": 0.5027,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.9981285929679871,
-      "rewards/margins": 0.7992982864379883,
-      "rewards/rejected": -1.7974268198013306,
+      "logits/chosen": 1.8378353118896484,
+      "logits/rejected": 2.7034897804260254,
+      "logps/chosen": -376.66876220703125,
+      "logps/rejected": -429.076904296875,
+      "loss": 0.5008,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.046795129776001,
+      "rewards/margins": 0.8175219297409058,
+      "rewards/rejected": -1.8643172979354858,
       "step": 460
     },
     {
       "epoch": 0.9832635983263598,
-      "grad_norm": 25.51430496068903,
+      "grad_norm": 25.63144929511813,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": 0.8643745183944702,
-      "logits/rejected": 1.9723937511444092,
-      "logps/chosen": -376.01934814453125,
-      "logps/rejected": -464.6620178222656,
-      "loss": 0.4911,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.9732152819633484,
-      "rewards/margins": 0.8496482968330383,
-      "rewards/rejected": -1.8228635787963867,
+      "logits/chosen": 1.5859147310256958,
+      "logits/rejected": 2.591291666030884,
+      "logps/chosen": -382.67071533203125,
+      "logps/rejected": -471.57861328125,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.0397284030914307,
+      "rewards/margins": 0.852301299571991,
+      "rewards/rejected": -1.8920300006866455,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.5400737299579955,
-      "train_runtime": 7416.7822,
-      "train_samples_per_second": 8.243,
-      "train_steps_per_second": 0.064
+      "train_loss": 0.5401068107852377,
+      "train_runtime": 7660.5652,
+      "train_samples_per_second": 7.98,
+      "train_steps_per_second": 0.062
     }
   ],
   "logging_steps": 10,