diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.9984301412872841,
   "eval_steps": 500,
-  "global_step": 1699,
+  "global_step": 159,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0005885815185403178,
-      "grad_norm": 32.780718251913974,
-      "learning_rate": 2.942907592701589e-10,
-      "logits/chosen": -2.603684186935425,
-      "logits/rejected": -2.7838211059570312,
-      "logps/chosen": -294.7889099121094,
-      "logps/rejected": -69.72528076171875,
+      "epoch": 0.006279434850863423,
+      "grad_norm": 10.468690716679266,
+      "learning_rate": 3.125e-08,
+      "logits/chosen": -2.8471853733062744,
+      "logits/rejected": -2.8164334297180176,
+      "logps/chosen": -315.592529296875,
+      "logps/rejected": -230.47390747070312,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,2552 +24,242 @@
       "step": 1
     },
     {
-      "epoch": 0.005885815185403178,
-      "grad_norm": 30.421018457904477,
-      "learning_rate": 2.9429075927015888e-09,
-      "logits/chosen": -2.8964638710021973,
-      "logits/rejected": -2.901930809020996,
-      "logps/chosen": -331.2449645996094,
-      "logps/rejected": -301.0341796875,
-      "loss": 0.6933,
-      "rewards/accuracies": 0.2222222238779068,
-      "rewards/chosen": -0.0001181578409159556,
-      "rewards/margins": -0.0006874248501844704,
-      "rewards/rejected": 0.0005692669074051082,
+      "epoch": 0.06279434850863422,
+      "grad_norm": 9.878509369466363,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.8302619457244873,
+      "logits/rejected": -2.8141226768493652,
+      "logps/chosen": -265.6414794921875,
+      "logps/rejected": -265.56695556640625,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4548611044883728,
+      "rewards/chosen": 0.0005584080354310572,
+      "rewards/margins": 0.0001245423045475036,
+      "rewards/rejected": 0.00043386564357206225,
       "step": 10
     },
     {
-      "epoch": 0.011771630370806356,
-      "grad_norm": 39.77348485571547,
-      "learning_rate": 5.8858151854031776e-09,
-      "logits/chosen": -2.890000581741333,
-      "logits/rejected": -2.8872923851013184,
-      "logps/chosen": -299.2838439941406,
-      "logps/rejected": -242.45925903320312,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.5333333611488342,
-      "rewards/chosen": 0.00012406618043314666,
-      "rewards/margins": 0.0003421924775466323,
-      "rewards/rejected": -0.0002181262243539095,
+      "epoch": 0.12558869701726844,
+      "grad_norm": 10.278326132890863,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": -2.8090524673461914,
+      "logits/rejected": -2.788933277130127,
+      "logps/chosen": -253.39077758789062,
+      "logps/rejected": -246.2702178955078,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.019792597740888596,
+      "rewards/margins": 0.011014475487172604,
+      "rewards/rejected": 0.00877812597900629,
       "step": 20
     },
     {
-      "epoch": 0.017657445556209534,
-      "grad_norm": 37.15226364119724,
-      "learning_rate": 8.828722778104766e-09,
-      "logits/chosen": -2.871821403503418,
-      "logits/rejected": -2.821002721786499,
-      "logps/chosen": -275.4599609375,
-      "logps/rejected": -248.410888671875,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.3999999761581421,
-      "rewards/chosen": 7.882448699092492e-05,
-      "rewards/margins": -0.0011783873196691275,
-      "rewards/rejected": 0.0012572116684168577,
+      "epoch": 0.18838304552590268,
+      "grad_norm": 11.048616082787978,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": -2.763706684112549,
+      "logits/rejected": -2.75482439994812,
+      "logps/chosen": -270.31256103515625,
+      "logps/rejected": -236.15945434570312,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.038934122771024704,
+      "rewards/margins": 0.04564380645751953,
+      "rewards/rejected": -0.006709681358188391,
       "step": 30
     },
     {
-      "epoch": 0.023543260741612712,
-      "grad_norm": 28.986298484577823,
-      "learning_rate": 1.1771630370806355e-08,
-      "logits/chosen": -2.8257861137390137,
-      "logits/rejected": -2.782503366470337,
-      "logps/chosen": -278.97601318359375,
-      "logps/rejected": -273.42205810546875,
-      "loss": 0.6937,
-      "rewards/accuracies": 0.30000001192092896,
-      "rewards/chosen": -0.0018755996134132147,
-      "rewards/margins": -0.0013799888547509909,
-      "rewards/rejected": -0.0004956105258315802,
+      "epoch": 0.25117739403453687,
+      "grad_norm": 11.185166495621557,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": -2.7219185829162598,
+      "logits/rejected": -2.701842784881592,
+      "logps/chosen": -254.3495330810547,
+      "logps/rejected": -254.6168212890625,
+      "loss": 0.6605,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": -0.03600288927555084,
+      "rewards/margins": 0.07350684702396393,
+      "rewards/rejected": -0.10950972884893417,
       "step": 40
     },
     {
-      "epoch": 0.02942907592701589,
-      "grad_norm": 28.82026354510354,
-      "learning_rate": 1.4714537963507944e-08,
-      "logits/chosen": -2.855421543121338,
-      "logits/rejected": -2.8377156257629395,
-      "logps/chosen": -253.5948944091797,
-      "logps/rejected": -274.3114318847656,
-      "loss": 0.6937,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.0006353737553581595,
-      "rewards/margins": -0.0002903261920437217,
-      "rewards/rejected": -0.00034504756331443787,
+      "epoch": 0.3139717425431711,
+      "grad_norm": 13.063361771402542,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": -2.6782164573669434,
+      "logits/rejected": -2.6529879570007324,
+      "logps/chosen": -274.06939697265625,
+      "logps/rejected": -282.3935546875,
+      "loss": 0.634,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": -0.07310701161623001,
+      "rewards/margins": 0.18823260068893433,
+      "rewards/rejected": -0.26133960485458374,
       "step": 50
     },
     {
-      "epoch": 0.03531489111241907,
-      "grad_norm": 30.822584821190027,
-      "learning_rate": 1.7657445556209532e-08,
-      "logits/chosen": -2.824986696243286,
-      "logits/rejected": -2.8068385124206543,
-      "logps/chosen": -330.98846435546875,
-      "logps/rejected": -329.9060974121094,
-      "loss": 0.6937,
-      "rewards/accuracies": 0.4666667580604553,
-      "rewards/chosen": 0.00020488860900513828,
-      "rewards/margins": -0.0003379290283191949,
-      "rewards/rejected": 0.000542817753739655,
+      "epoch": 0.37676609105180536,
+      "grad_norm": 15.023213763564414,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": -2.7063775062561035,
+      "logits/rejected": -2.677776336669922,
+      "logps/chosen": -301.80023193359375,
+      "logps/rejected": -276.10565185546875,
+      "loss": 0.6256,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": -0.1930098533630371,
+      "rewards/margins": 0.21952715516090393,
+      "rewards/rejected": -0.41253700852394104,
       "step": 60
     },
     {
-      "epoch": 0.04120070629782225,
-      "grad_norm": 32.409000763227326,
-      "learning_rate": 2.0600353148911124e-08,
-      "logits/chosen": -2.6729750633239746,
-      "logits/rejected": -2.7752492427825928,
-      "logps/chosen": -288.5548400878906,
-      "logps/rejected": -286.0675964355469,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.0015738470247015357,
-      "rewards/margins": -0.001438200124539435,
-      "rewards/rejected": -0.0001356468565063551,
+      "epoch": 0.43956043956043955,
+      "grad_norm": 15.201166100431667,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": -2.7157435417175293,
+      "logits/rejected": -2.6909706592559814,
+      "logps/chosen": -312.8190612792969,
+      "logps/rejected": -312.00616455078125,
+      "loss": 0.6094,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.30874696373939514,
+      "rewards/margins": 0.24815471470355988,
+      "rewards/rejected": -0.556901752948761,
       "step": 70
     },
     {
-      "epoch": 0.047086521483225424,
-      "grad_norm": 28.66094375035891,
-      "learning_rate": 2.354326074161271e-08,
-      "logits/chosen": -2.8610806465148926,
-      "logits/rejected": -2.809129476547241,
-      "logps/chosen": -202.50711059570312,
-      "logps/rejected": -256.8191833496094,
-      "loss": 0.6933,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.0010831685503944755,
-      "rewards/margins": -0.0023852435406297445,
-      "rewards/rejected": 0.001302075106650591,
+      "epoch": 0.5023547880690737,
+      "grad_norm": 22.235789809346436,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": -2.711451292037964,
+      "logits/rejected": -2.6855673789978027,
+      "logps/chosen": -325.4831237792969,
+      "logps/rejected": -324.86187744140625,
+      "loss": 0.5897,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.31690555810928345,
+      "rewards/margins": 0.35460132360458374,
+      "rewards/rejected": -0.6715068817138672,
       "step": 80
     },
     {
-      "epoch": 0.052972336668628606,
-      "grad_norm": 31.002618375475574,
-      "learning_rate": 2.6486168334314303e-08,
-      "logits/chosen": -2.9170703887939453,
-      "logits/rejected": -2.8157236576080322,
-      "logps/chosen": -280.6930236816406,
-      "logps/rejected": -256.5555725097656,
-      "loss": 0.6924,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0004723949823528528,
-      "rewards/margins": 0.0002984645252581686,
-      "rewards/rejected": 0.00017393044254276901,
+      "epoch": 0.565149136577708,
+      "grad_norm": 18.534325691706606,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": -2.72774338722229,
+      "logits/rejected": -2.7118916511535645,
+      "logps/chosen": -298.873779296875,
+      "logps/rejected": -330.8174133300781,
+      "loss": 0.5713,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.43646472692489624,
+      "rewards/margins": 0.34521809220314026,
+      "rewards/rejected": -0.7816828489303589,
       "step": 90
     },
     {
-      "epoch": 0.05885815185403178,
-      "grad_norm": 35.75851601438966,
-      "learning_rate": 2.942907592701589e-08,
-      "logits/chosen": -2.882338285446167,
-      "logits/rejected": -2.8756601810455322,
-      "logps/chosen": -284.1132507324219,
-      "logps/rejected": -261.82562255859375,
-      "loss": 0.693,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.0010607184376567602,
-      "rewards/margins": 0.0015401893761008978,
-      "rewards/rejected": -0.0004794707929249853,
+      "epoch": 0.6279434850863422,
+      "grad_norm": 19.673309131968132,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": -2.6677684783935547,
+      "logits/rejected": -2.6389312744140625,
+      "logps/chosen": -322.52081298828125,
+      "logps/rejected": -328.7558288574219,
+      "loss": 0.5875,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.37618938088417053,
+      "rewards/margins": 0.3784511685371399,
+      "rewards/rejected": -0.7546405792236328,
       "step": 100
     },
     {
-      "epoch": 0.06474396703943497,
-      "grad_norm": 26.830290079245987,
-      "learning_rate": 3.2371983519717484e-08,
-      "logits/chosen": -2.8224923610687256,
-      "logits/rejected": -2.786034107208252,
-      "logps/chosen": -213.18856811523438,
-      "logps/rejected": -209.486572265625,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.5000000596046448,
-      "rewards/chosen": 0.0004384171334095299,
-      "rewards/margins": 0.000683752354234457,
-      "rewards/rejected": -0.00024533524992875755,
+      "epoch": 0.6907378335949764,
+      "grad_norm": 17.53131220916881,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": -2.6724138259887695,
+      "logits/rejected": -2.6650776863098145,
+      "logps/chosen": -285.9563293457031,
+      "logps/rejected": -335.96673583984375,
+      "loss": 0.5746,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": -0.47776883840560913,
+      "rewards/margins": 0.4206571578979492,
+      "rewards/rejected": -0.8984260559082031,
       "step": 110
     },
     {
-      "epoch": 0.07062978222483814,
-      "grad_norm": 33.59257452712036,
-      "learning_rate": 3.5314891112419064e-08,
-      "logits/chosen": -2.7929792404174805,
-      "logits/rejected": -2.787545919418335,
-      "logps/chosen": -296.0240478515625,
-      "logps/rejected": -260.3634033203125,
-      "loss": 0.6929,
-      "rewards/accuracies": 0.4333333373069763,
-      "rewards/chosen": 8.877464279066771e-05,
-      "rewards/margins": -0.0005403327522799373,
-      "rewards/rejected": 0.000629107526037842,
+      "epoch": 0.7535321821036107,
+      "grad_norm": 20.93530860525305,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": -2.7068216800689697,
+      "logits/rejected": -2.685987949371338,
+      "logps/chosen": -314.00042724609375,
+      "logps/rejected": -346.78546142578125,
+      "loss": 0.5883,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5427230596542358,
+      "rewards/margins": 0.36296552419662476,
+      "rewards/rejected": -0.9056886434555054,
       "step": 120
     },
     {
-      "epoch": 0.07651559741024132,
-      "grad_norm": 31.167055741749806,
-      "learning_rate": 3.8257798705120656e-08,
-      "logits/chosen": -2.929863452911377,
-      "logits/rejected": -2.8793587684631348,
-      "logps/chosen": -192.31356811523438,
-      "logps/rejected": -197.66647338867188,
-      "loss": 0.6926,
-      "rewards/accuracies": 0.5000000596046448,
-      "rewards/chosen": 0.001261854893527925,
-      "rewards/margins": 0.00017420703079551458,
-      "rewards/rejected": 0.0010876477463170886,
+      "epoch": 0.8163265306122449,
+      "grad_norm": 27.215171241370864,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": -2.725468635559082,
+      "logits/rejected": -2.6890971660614014,
+      "logps/chosen": -312.6821594238281,
+      "logps/rejected": -338.29144287109375,
+      "loss": 0.5549,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": -0.45934683084487915,
+      "rewards/margins": 0.4836532175540924,
+      "rewards/rejected": -0.9429999589920044,
       "step": 130
     },
     {
-      "epoch": 0.0824014125956445,
-      "grad_norm": 25.597516159244545,
-      "learning_rate": 4.120070629782225e-08,
-      "logits/chosen": -2.818716526031494,
-      "logits/rejected": -2.8491830825805664,
-      "logps/chosen": -267.6719970703125,
-      "logps/rejected": -269.23138427734375,
-      "loss": 0.6924,
-      "rewards/accuracies": 0.5666667222976685,
-      "rewards/chosen": 0.002183220349252224,
-      "rewards/margins": 0.0013612338807433844,
-      "rewards/rejected": 0.0008219865849241614,
+      "epoch": 0.8791208791208791,
+      "grad_norm": 21.30706575112927,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": -2.7268600463867188,
+      "logits/rejected": -2.698526382446289,
+      "logps/chosen": -334.44989013671875,
+      "logps/rejected": -353.6800842285156,
+      "loss": 0.5646,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.4807310998439789,
+      "rewards/margins": 0.43861979246139526,
+      "rewards/rejected": -0.9193509221076965,
       "step": 140
     },
     {
-      "epoch": 0.08828722778104768,
-      "grad_norm": 30.754492049125638,
-      "learning_rate": 4.414361389052384e-08,
-      "logits/chosen": -2.807731866836548,
-      "logits/rejected": -2.82275652885437,
-      "logps/chosen": -256.2826843261719,
-      "logps/rejected": -216.8761749267578,
-      "loss": 0.6922,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": 0.0029171775095164776,
-      "rewards/margins": 0.0019290171330794692,
-      "rewards/rejected": 0.0009881604928523302,
+      "epoch": 0.9419152276295133,
+      "grad_norm": 20.93813177233855,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": -2.690723419189453,
+      "logits/rejected": -2.6735761165618896,
+      "logps/chosen": -309.25262451171875,
+      "logps/rejected": -340.45733642578125,
+      "loss": 0.5617,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.47407713532447815,
+      "rewards/margins": 0.4926603436470032,
+      "rewards/rejected": -0.9667374491691589,
       "step": 150
     },
     {
-      "epoch": 0.09417304296645085,
-      "grad_norm": 33.216197696590626,
-      "learning_rate": 4.708652148322542e-08,
-      "logits/chosen": -2.7577600479125977,
-      "logits/rejected": -2.686497449874878,
-      "logps/chosen": -271.95977783203125,
-      "logps/rejected": -216.9535675048828,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0015044666361063719,
-      "rewards/margins": -0.0006265610572881997,
-      "rewards/rejected": 0.0021310276351869106,
-      "step": 160
-    },
-    {
-      "epoch": 0.10005885815185403,
-      "grad_norm": 31.85341360598595,
-      "learning_rate": 5.002942907592701e-08,
-      "logits/chosen": -2.860220432281494,
-      "logits/rejected": -2.824690580368042,
-      "logps/chosen": -229.2205047607422,
-      "logps/rejected": -216.49691772460938,
-      "loss": 0.6916,
-      "rewards/accuracies": 0.7333332896232605,
-      "rewards/chosen": 0.005175250582396984,
-      "rewards/margins": 0.004574229009449482,
-      "rewards/rejected": 0.0006010220386087894,
-      "step": 170
-    },
-    {
-      "epoch": 0.10594467333725721,
-      "grad_norm": 33.35020421307099,
-      "learning_rate": 5.2972336668628605e-08,
-      "logits/chosen": -2.8476788997650146,
-      "logits/rejected": -2.8345282077789307,
-      "logps/chosen": -276.9328918457031,
-      "logps/rejected": -297.92138671875,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": 0.002554533304646611,
-      "rewards/margins": 0.002964629326015711,
-      "rewards/rejected": -0.0004100958467461169,
-      "step": 180
-    },
-    {
-      "epoch": 0.11183048852266039,
-      "grad_norm": 27.907028115997573,
-      "learning_rate": 5.591524426133019e-08,
-      "logits/chosen": -2.8730475902557373,
-      "logits/rejected": -2.839439868927002,
-      "logps/chosen": -272.5675048828125,
-      "logps/rejected": -264.9402770996094,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": 0.002410594839602709,
-      "rewards/margins": 0.000803423929028213,
-      "rewards/rejected": 0.0016071706777438521,
-      "step": 190
-    },
-    {
-      "epoch": 0.11771630370806356,
-      "grad_norm": 29.481219977820295,
-      "learning_rate": 5.885815185403178e-08,
-      "logits/chosen": -2.9408724308013916,
-      "logits/rejected": -2.815699338912964,
-      "logps/chosen": -310.2392883300781,
-      "logps/rejected": -236.49404907226562,
-      "loss": 0.6904,
-      "rewards/accuracies": 0.6666666269302368,
-      "rewards/chosen": 0.006228293292224407,
-      "rewards/margins": 0.005451700650155544,
-      "rewards/rejected": 0.0007765936315990984,
-      "step": 200
-    },
-    {
-      "epoch": 0.12360211889346674,
-      "grad_norm": 35.046475552370886,
-      "learning_rate": 6.180105944673337e-08,
-      "logits/chosen": -2.6355643272399902,
-      "logits/rejected": -2.668175458908081,
-      "logps/chosen": -314.09619140625,
-      "logps/rejected": -340.42645263671875,
-      "loss": 0.6923,
-      "rewards/accuracies": 0.5666666626930237,
-      "rewards/chosen": 0.010181600227952003,
-      "rewards/margins": 0.004080074839293957,
-      "rewards/rejected": 0.006101525854319334,
-      "step": 210
-    },
-    {
-      "epoch": 0.12948793407886994,
-      "grad_norm": 30.183227607657507,
-      "learning_rate": 6.474396703943497e-08,
-      "logits/chosen": -2.922337293624878,
-      "logits/rejected": -2.9165425300598145,
-      "logps/chosen": -311.33917236328125,
-      "logps/rejected": -271.88800048828125,
-      "loss": 0.6911,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.008037504740059376,
-      "rewards/margins": 0.006833515129983425,
-      "rewards/rejected": 0.0012039883295074105,
-      "step": 220
-    },
-    {
-      "epoch": 0.1353737492642731,
-      "grad_norm": 28.651320388398528,
-      "learning_rate": 6.768687463213654e-08,
-      "logits/chosen": -2.764054536819458,
-      "logits/rejected": -2.791746139526367,
-      "logps/chosen": -342.35638427734375,
-      "logps/rejected": -269.53033447265625,
-      "loss": 0.6902,
-      "rewards/accuracies": 0.8333333730697632,
-      "rewards/chosen": 0.011332266964018345,
-      "rewards/margins": 0.010512511245906353,
-      "rewards/rejected": 0.0008197550778277218,
-      "step": 230
-    },
-    {
-      "epoch": 0.14125956444967627,
-      "grad_norm": 30.194472525654817,
-      "learning_rate": 7.062978222483813e-08,
-      "logits/chosen": -2.9200429916381836,
-      "logits/rejected": -2.8475396633148193,
-      "logps/chosen": -330.425537109375,
-      "logps/rejected": -282.93597412109375,
-      "loss": 0.6887,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": 0.015913909301161766,
-      "rewards/margins": 0.013330700807273388,
-      "rewards/rejected": 0.0025832061655819416,
-      "step": 240
-    },
-    {
-      "epoch": 0.14714537963507945,
-      "grad_norm": 49.29351755297091,
-      "learning_rate": 7.357268981753973e-08,
-      "logits/chosen": -2.809460401535034,
-      "logits/rejected": -2.847788095474243,
-      "logps/chosen": -228.82962036132812,
-      "logps/rejected": -260.50067138671875,
-      "loss": 0.6886,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.010601197369396687,
-      "rewards/margins": 0.0034695893991738558,
-      "rewards/rejected": 0.007131607737392187,
-      "step": 250
-    },
-    {
-      "epoch": 0.15303119482048264,
-      "grad_norm": 35.22606986399358,
-      "learning_rate": 7.651559741024131e-08,
-      "logits/chosen": -2.829927921295166,
-      "logits/rejected": -2.7731313705444336,
-      "logps/chosen": -368.4117736816406,
-      "logps/rejected": -278.7731018066406,
-      "loss": 0.6901,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": 0.014880454167723656,
-      "rewards/margins": 0.008975975215435028,
-      "rewards/rejected": 0.00590447848662734,
-      "step": 260
-    },
-    {
-      "epoch": 0.15891701000588582,
-      "grad_norm": 31.079626356652042,
-      "learning_rate": 7.945850500294291e-08,
-      "logits/chosen": -2.8230886459350586,
-      "logits/rejected": -2.8253014087677,
-      "logps/chosen": -246.8427276611328,
-      "logps/rejected": -257.07440185546875,
-      "loss": 0.6887,
-      "rewards/accuracies": 0.6666666269302368,
-      "rewards/chosen": 0.011582859791815281,
-      "rewards/margins": 0.00953072588890791,
-      "rewards/rejected": 0.002052133437246084,
-      "step": 270
-    },
-    {
-      "epoch": 0.164802825191289,
-      "grad_norm": 34.16689629310999,
-      "learning_rate": 8.24014125956445e-08,
-      "logits/chosen": -2.790585994720459,
-      "logits/rejected": -2.741240978240967,
-      "logps/chosen": -194.43751525878906,
-      "logps/rejected": -246.21109008789062,
-      "loss": 0.6856,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.011640877462923527,
-      "rewards/margins": 0.013520603068172932,
-      "rewards/rejected": -0.0018797263037413359,
-      "step": 280
-    },
-    {
-      "epoch": 0.17068864037669218,
-      "grad_norm": 26.650476040593702,
-      "learning_rate": 8.534432018834608e-08,
-      "logits/chosen": -2.877924680709839,
-      "logits/rejected": -2.9167160987854004,
-      "logps/chosen": -200.64999389648438,
-      "logps/rejected": -239.98623657226562,
-      "loss": 0.6864,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.015060337260365486,
-      "rewards/margins": 0.004761075135320425,
-      "rewards/rejected": 0.0102992607280612,
-      "step": 290
-    },
-    {
-      "epoch": 0.17657445556209536,
-      "grad_norm": 37.84975273963207,
-      "learning_rate": 8.828722778104768e-08,
-      "logits/chosen": -2.7102274894714355,
-      "logits/rejected": -2.723665475845337,
-      "logps/chosen": -241.3960418701172,
-      "logps/rejected": -201.9961395263672,
-      "loss": 0.6829,
-      "rewards/accuracies": 0.7333332896232605,
-      "rewards/chosen": 0.023427072912454605,
-      "rewards/margins": 0.020558174699544907,
-      "rewards/rejected": 0.002868895884603262,
-      "step": 300
-    },
-    {
-      "epoch": 0.18246027074749852,
-      "grad_norm": 32.582230477501014,
-      "learning_rate": 9.123013537374925e-08,
-      "logits/chosen": -2.771923303604126,
-      "logits/rejected": -2.755645275115967,
-      "logps/chosen": -236.4650421142578,
-      "logps/rejected": -176.6122589111328,
-      "loss": 0.6846,
-      "rewards/accuracies": 0.8333331942558289,
-      "rewards/chosen": 0.03548317030072212,
-      "rewards/margins": 0.026407241821289062,
-      "rewards/rejected": 0.00907592847943306,
-      "step": 310
-    },
-    {
-      "epoch": 0.1883460859329017,
-      "grad_norm": 35.39155087276743,
-      "learning_rate": 9.417304296645084e-08,
-      "logits/chosen": -2.809967517852783,
-      "logits/rejected": -2.7024905681610107,
-      "logps/chosen": -267.1817932128906,
-      "logps/rejected": -216.0163116455078,
-      "loss": 0.6802,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.027452057227492332,
-      "rewards/margins": 0.02637614868581295,
-      "rewards/rejected": 0.0010759088909253478,
-      "step": 320
-    },
-    {
-      "epoch": 0.19423190111830488,
-      "grad_norm": 25.00506965748009,
-      "learning_rate": 9.711595055915244e-08,
-      "logits/chosen": -2.8656439781188965,
-      "logits/rejected": -2.797421932220459,
-      "logps/chosen": -223.9487762451172,
-      "logps/rejected": -238.2415313720703,
-      "loss": 0.687,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.018507182598114014,
-      "rewards/margins": 0.011660409159958363,
-      "rewards/rejected": 0.006846773438155651,
-      "step": 330
-    },
-    {
-      "epoch": 0.20011771630370806,
-      "grad_norm": 31.48968715365344,
-      "learning_rate": 1.0005885815185403e-07,
-      "logits/chosen": -2.8663744926452637,
-      "logits/rejected": -2.829288959503174,
-      "logps/chosen": -227.17904663085938,
-      "logps/rejected": -254.73300170898438,
-      "loss": 0.6882,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.03473003953695297,
-      "rewards/margins": -0.002185368211939931,
-      "rewards/rejected": 0.036915406584739685,
-      "step": 340
-    },
-    {
-      "epoch": 0.20600353148911124,
-      "grad_norm": 33.13341184823187,
-      "learning_rate": 1.0300176574455561e-07,
-      "logits/chosen": -2.8728482723236084,
-      "logits/rejected": -2.849656581878662,
-      "logps/chosen": -232.64306640625,
-      "logps/rejected": -189.30641174316406,
-      "loss": 0.6844,
-      "rewards/accuracies": 0.5999999642372131,
-      "rewards/chosen": 0.025649067014455795,
-      "rewards/margins": 0.01318146288394928,
-      "rewards/rejected": 0.01246760506182909,
-      "step": 350
-    },
-    {
-      "epoch": 0.21188934667451442,
-      "grad_norm": 33.57559926375768,
-      "learning_rate": 1.0594467333725721e-07,
-      "logits/chosen": -2.94728422164917,
-      "logits/rejected": -2.8343241214752197,
-      "logps/chosen": -330.82861328125,
-      "logps/rejected": -255.1486053466797,
-      "loss": 0.6704,
-      "rewards/accuracies": 0.7999999523162842,
-      "rewards/chosen": 0.060690779238939285,
-      "rewards/margins": 0.051527559757232666,
-      "rewards/rejected": 0.009163224138319492,
-      "step": 360
-    },
-    {
-      "epoch": 0.2177751618599176,
-      "grad_norm": 30.406704666801055,
-      "learning_rate": 1.088875809299588e-07,
-      "logits/chosen": -2.847501039505005,
-      "logits/rejected": -2.859219551086426,
-      "logps/chosen": -226.4930877685547,
-      "logps/rejected": -411.73291015625,
-      "loss": 0.6794,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": 0.029315758496522903,
-      "rewards/margins": 0.03848721459507942,
-      "rewards/rejected": -0.009171457961201668,
-      "step": 370
-    },
-    {
-      "epoch": 0.22366097704532079,
-      "grad_norm": 32.000980950184385,
-      "learning_rate": 1.1183048852266038e-07,
-      "logits/chosen": -2.750246047973633,
-      "logits/rejected": -2.683522939682007,
-      "logps/chosen": -312.06695556640625,
-      "logps/rejected": -251.6304931640625,
-      "loss": 0.6762,
-      "rewards/accuracies": 0.6666666865348816,
-      "rewards/chosen": 0.029916947707533836,
-      "rewards/margins": 0.033031001687049866,
-      "rewards/rejected": -0.003114058170467615,
-      "step": 380
-    },
-    {
-      "epoch": 0.22954679223072397,
-      "grad_norm": 34.96067667192858,
-      "learning_rate": 1.1477339611536198e-07,
-      "logits/chosen": -2.855989694595337,
-      "logits/rejected": -2.815114974975586,
-      "logps/chosen": -262.18450927734375,
-      "logps/rejected": -189.57838439941406,
-      "loss": 0.6824,
-      "rewards/accuracies": 0.7333333492279053,
-      "rewards/chosen": 0.025654960423707962,
-      "rewards/margins": 0.0397619903087616,
-      "rewards/rejected": -0.01410702895373106,
-      "step": 390
-    },
-    {
-      "epoch": 0.23543260741612712,
-      "grad_norm": 30.784886423492903,
-      "learning_rate": 1.1771630370806355e-07,
-      "logits/chosen": -2.8241119384765625,
-      "logits/rejected": -2.836333751678467,
-      "logps/chosen": -233.1301727294922,
-      "logps/rejected": -277.805419921875,
-      "loss": 0.6805,
-      "rewards/accuracies": 0.7333332896232605,
-      "rewards/chosen": 0.03232261911034584,
-      "rewards/margins": 0.03198341652750969,
-      "rewards/rejected": 0.00033920182613655925,
-      "step": 400
-    },
-    {
-      "epoch": 0.2413184226015303,
-      "grad_norm": 27.065262812921755,
-      "learning_rate": 1.2065921130076514e-07,
-      "logits/chosen": -2.756892442703247,
-      "logits/rejected": -2.6843674182891846,
-      "logps/chosen": -287.8220520019531,
-      "logps/rejected": -222.5053253173828,
-      "loss": 0.67,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.03421301394701004,
-      "rewards/margins": 0.054109979420900345,
-      "rewards/rejected": -0.019896967336535454,
-      "step": 410
-    },
-    {
-      "epoch": 0.24720423778693348,
-      "grad_norm": 33.86568434240492,
-      "learning_rate": 1.2360211889346674e-07,
-      "logits/chosen": -2.8926842212677,
-      "logits/rejected": -2.817612648010254,
-      "logps/chosen": -268.4040222167969,
-      "logps/rejected": -255.06655883789062,
-      "loss": 0.6698,
-      "rewards/accuracies": 0.6666666865348816,
-      "rewards/chosen": 0.014465409331023693,
-      "rewards/margins": 0.02410224825143814,
-      "rewards/rejected": -0.009636839851737022,
-      "step": 420
-    },
-    {
-      "epoch": 0.25309005297233667,
-      "grad_norm": 30.66902266053471,
-      "learning_rate": 1.2654502648616834e-07,
-      "logits/chosen": -2.7713985443115234,
-      "logits/rejected": -2.76814341545105,
-      "logps/chosen": -209.8004913330078,
-      "logps/rejected": -219.0537567138672,
-      "loss": 0.6823,
-      "rewards/accuracies": 0.7333332896232605,
-      "rewards/chosen": 0.0189804807305336,
-      "rewards/margins": 0.026658236980438232,
-      "rewards/rejected": -0.007677753455936909,
-      "step": 430
-    },
-    {
-      "epoch": 0.2589758681577399,
-      "grad_norm": 37.33663950349735,
-      "learning_rate": 1.2948793407886994e-07,
-      "logits/chosen": -2.762784481048584,
-      "logits/rejected": -2.707094669342041,
-      "logps/chosen": -250.97177124023438,
-      "logps/rejected": -232.8260040283203,
-      "loss": 0.6651,
-      "rewards/accuracies": 0.7000000476837158,
-      "rewards/chosen": 0.03512561693787575,
-      "rewards/margins": 0.09343008697032928,
-      "rewards/rejected": -0.05830447003245354,
-      "step": 440
-    },
-    {
-      "epoch": 0.26486168334314303,
-      "grad_norm": 38.55541735391845,
-      "learning_rate": 1.324308416715715e-07,
-      "logits/chosen": -2.7505548000335693,
-      "logits/rejected": -2.790361166000366,
-      "logps/chosen": -261.1943664550781,
-      "logps/rejected": -230.3916473388672,
-      "loss": 0.6643,
-      "rewards/accuracies": 0.7666667103767395,
-      "rewards/chosen": 0.03301350399851799,
-      "rewards/margins": 0.08106649667024612,
-      "rewards/rejected": -0.04805298522114754,
-      "step": 450
-    },
-    {
-      "epoch": 0.2707474985285462,
-      "grad_norm": 30.70708391983573,
-      "learning_rate": 1.3537374926427308e-07,
-      "logits/chosen": -2.782029390335083,
-      "logits/rejected": -2.710545539855957,
-      "logps/chosen": -238.57211303710938,
-      "logps/rejected": -251.1454315185547,
-      "loss": 0.6818,
-      "rewards/accuracies": 0.5666666626930237,
-      "rewards/chosen": -0.005198891274631023,
-      "rewards/margins": 0.015401232056319714,
-      "rewards/rejected": -0.020600125193595886,
-      "step": 460
-    },
-    {
-      "epoch": 0.2766333137139494,
-      "grad_norm": 33.06918545223034,
-      "learning_rate": 1.3831665685697468e-07,
-      "logits/chosen": -2.7734148502349854,
-      "logits/rejected": -2.722306728363037,
-      "logps/chosen": -236.19290161132812,
-      "logps/rejected": -218.6464080810547,
-      "loss": 0.6545,
-      "rewards/accuracies": 0.5666666030883789,
-      "rewards/chosen": 0.004786822013556957,
-      "rewards/margins": 0.04201732203364372,
-      "rewards/rejected": -0.03723050281405449,
-      "step": 470
-    },
-    {
-      "epoch": 0.28251912889935255,
-      "grad_norm": 34.67042804510441,
-      "learning_rate": 1.4125956444967625e-07,
-      "logits/chosen": -2.847014904022217,
-      "logits/rejected": -2.759166717529297,
-      "logps/chosen": -291.9137878417969,
-      "logps/rejected": -247.39566040039062,
-      "loss": 0.6612,
-      "rewards/accuracies": 0.46666669845581055,
-      "rewards/chosen": -0.01318571250885725,
-      "rewards/margins": 0.02275732532143593,
-      "rewards/rejected": -0.03594304248690605,
-      "step": 480
-    },
-    {
-      "epoch": 0.28840494408475575,
-      "grad_norm": 41.62165688207727,
-      "learning_rate": 1.4420247204237788e-07,
-      "logits/chosen": -2.8126304149627686,
-      "logits/rejected": -2.8055615425109863,
-      "logps/chosen": -289.50091552734375,
-      "logps/rejected": -277.8962707519531,
-      "loss": 0.6614,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": 0.020298918709158897,
-      "rewards/margins": 0.0497608408331871,
-      "rewards/rejected": -0.02946191094815731,
-      "step": 490
-    },
-    {
-      "epoch": 0.2942907592701589,
-      "grad_norm": 36.97324400383161,
-      "learning_rate": 1.4714537963507945e-07,
-      "logits/chosen": -2.885690212249756,
-      "logits/rejected": -2.8348922729492188,
-      "logps/chosen": -307.3095703125,
-      "logps/rejected": -290.29180908203125,
-      "loss": 0.6509,
-      "rewards/accuracies": 0.7333332896232605,
-      "rewards/chosen": 0.024543384090065956,
-      "rewards/margins": 0.09910109639167786,
-      "rewards/rejected": -0.07455770671367645,
-      "step": 500
-    },
-    {
-      "epoch": 0.3001765744555621,
-      "grad_norm": 36.070076135435244,
-      "learning_rate": 1.5008828722778105e-07,
-      "logits/chosen": -2.842216968536377,
-      "logits/rejected": -2.820446491241455,
-      "logps/chosen": -256.1488342285156,
-      "logps/rejected": -247.92672729492188,
-      "loss": 0.6638,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -0.0209733247756958,
-      "rewards/margins": 0.04726904630661011,
-      "rewards/rejected": -0.0682423785328865,
-      "step": 510
-    },
-    {
-      "epoch": 0.30606238964096527,
-      "grad_norm": 42.70438629438365,
-      "learning_rate": 1.5303119482048262e-07,
-      "logits/chosen": -2.6641478538513184,
-      "logits/rejected": -2.66444993019104,
-      "logps/chosen": -259.26348876953125,
-      "logps/rejected": -269.11572265625,
-      "loss": 0.6363,
-      "rewards/accuracies": 0.6666666269302368,
-      "rewards/chosen": -0.010639377869665623,
-      "rewards/margins": 0.09347447007894516,
-      "rewards/rejected": -0.10411383956670761,
-      "step": 520
-    },
-    {
-      "epoch": 0.3119482048263684,
-      "grad_norm": 35.960710299811815,
-      "learning_rate": 1.559741024131842e-07,
-      "logits/chosen": -2.805729866027832,
-      "logits/rejected": -2.7076404094696045,
-      "logps/chosen": -255.88052368164062,
-      "logps/rejected": -234.35415649414062,
-      "loss": 0.6335,
-      "rewards/accuracies": 0.6666666269302368,
-      "rewards/chosen": -0.007552222814410925,
-      "rewards/margins": 0.11313612759113312,
-      "rewards/rejected": -0.12068834155797958,
-      "step": 530
-    },
-    {
-      "epoch": 0.31783402001177163,
-      "grad_norm": 44.82672193345242,
-      "learning_rate": 1.5891701000588582e-07,
-      "logits/chosen": -2.8400981426239014,
-      "logits/rejected": -2.8063666820526123,
-      "logps/chosen": -294.94451904296875,
-      "logps/rejected": -278.1987609863281,
-      "loss": 0.6372,
-      "rewards/accuracies": 0.6666666269302368,
-      "rewards/chosen": -0.08216958492994308,
-      "rewards/margins": 0.09824585914611816,
-      "rewards/rejected": -0.18041543662548065,
-      "step": 540
-    },
-    {
-      "epoch": 0.3237198351971748,
-      "grad_norm": 35.13933484391953,
-      "learning_rate": 1.618599175985874e-07,
-      "logits/chosen": -2.7300307750701904,
-      "logits/rejected": -2.7713351249694824,
-      "logps/chosen": -230.0304718017578,
-      "logps/rejected": -262.08721923828125,
-      "loss": 0.6309,
-      "rewards/accuracies": 0.7333332896232605,
-      "rewards/chosen": -0.08895792812108994,
-      "rewards/margins": 0.13027846813201904,
-      "rewards/rejected": -0.2192363739013672,
-      "step": 550
-    },
-    {
-      "epoch": 0.329605650382578,
-      "grad_norm": 52.54257576269853,
-      "learning_rate": 1.64802825191289e-07,
-      "logits/chosen": -2.753162145614624,
-      "logits/rejected": -2.768341541290283,
-      "logps/chosen": -252.78878784179688,
-      "logps/rejected": -274.1339111328125,
-      "loss": 0.6457,
-      "rewards/accuracies": 0.7333333492279053,
-      "rewards/chosen": -0.1532047688961029,
-      "rewards/margins": 0.1562652289867401,
-      "rewards/rejected": -0.309469997882843,
-      "step": 560
-    },
-    {
-      "epoch": 0.33549146556798115,
-      "grad_norm": 38.27091133745243,
-      "learning_rate": 1.6774573278399057e-07,
-      "logits/chosen": -2.804091215133667,
-      "logits/rejected": -2.776109457015991,
-      "logps/chosen": -287.103271484375,
-      "logps/rejected": -222.2604217529297,
-      "loss": 0.6418,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.04693541303277016,
-      "rewards/margins": 0.12523198127746582,
-      "rewards/rejected": -0.17216739058494568,
-      "step": 570
-    },
-    {
-      "epoch": 0.34137728075338436,
-      "grad_norm": 46.61649976864576,
-      "learning_rate": 1.7068864037669217e-07,
-      "logits/chosen": -2.802682399749756,
-      "logits/rejected": -2.7890288829803467,
-      "logps/chosen": -285.7334899902344,
-      "logps/rejected": -294.05181884765625,
-      "loss": 0.6359,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.16137059032917023,
-      "rewards/margins": 0.1200031265616417,
-      "rewards/rejected": -0.2813737392425537,
-      "step": 580
-    },
-    {
-      "epoch": 0.3472630959387875,
-      "grad_norm": 38.73511849094784,
-      "learning_rate": 1.7363154796939374e-07,
-      "logits/chosen": -2.7510268688201904,
-      "logits/rejected": -2.6245083808898926,
-      "logps/chosen": -265.32757568359375,
-      "logps/rejected": -240.0128631591797,
-      "loss": 0.6388,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.2014482468366623,
-      "rewards/margins": 0.0063117193058133125,
-      "rewards/rejected": -0.20775997638702393,
-      "step": 590
-    },
-    {
-      "epoch": 0.3531489111241907,
-      "grad_norm": 33.48738056386425,
-      "learning_rate": 1.7657445556209536e-07,
-      "logits/chosen": -2.8242006301879883,
-      "logits/rejected": -2.828608274459839,
-      "logps/chosen": -249.8431396484375,
-      "logps/rejected": -261.53948974609375,
-      "loss": 0.6199,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -0.0753629207611084,
-      "rewards/margins": 0.17823639512062073,
-      "rewards/rejected": -0.25359928607940674,
-      "step": 600
-    },
-    {
-      "epoch": 0.3590347263095939,
-      "grad_norm": 44.06827645574314,
-      "learning_rate": 1.7951736315479694e-07,
-      "logits/chosen": -2.652758836746216,
-      "logits/rejected": -2.7120182514190674,
-      "logps/chosen": -283.7317810058594,
-      "logps/rejected": -316.36004638671875,
-      "loss": 0.6189,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.2105875015258789,
-      "rewards/margins": 0.14809870719909668,
-      "rewards/rejected": -0.35868626832962036,
-      "step": 610
-    },
-    {
-      "epoch": 0.36492054149499703,
-      "grad_norm": 77.88854058346173,
-      "learning_rate": 1.824602707474985e-07,
-      "logits/chosen": -2.792132616043091,
-      "logits/rejected": -2.735396146774292,
-      "logps/chosen": -308.66094970703125,
-      "logps/rejected": -297.17120361328125,
-      "loss": 0.615,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.38594353199005127,
-      "rewards/margins": 0.13582220673561096,
-      "rewards/rejected": -0.5217657685279846,
-      "step": 620
-    },
-    {
-      "epoch": 0.37080635668040024,
-      "grad_norm": 51.14005626306302,
-      "learning_rate": 1.854031783402001e-07,
-      "logits/chosen": -2.6652121543884277,
-      "logits/rejected": -2.6964430809020996,
-      "logps/chosen": -246.6635284423828,
-      "logps/rejected": -252.4509735107422,
-      "loss": 0.6434,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.09110568463802338,
-      "rewards/margins": 0.1668696105480194,
-      "rewards/rejected": -0.2579752802848816,
-      "step": 630
-    },
-    {
-      "epoch": 0.3766921718658034,
-      "grad_norm": 51.12058774641213,
-      "learning_rate": 1.8834608593290168e-07,
-      "logits/chosen": -2.837921380996704,
-      "logits/rejected": -2.8364148139953613,
-      "logps/chosen": -307.8941955566406,
-      "logps/rejected": -311.6710205078125,
-      "loss": 0.6426,
-      "rewards/accuracies": 0.5333333611488342,
-      "rewards/chosen": -0.19721655547618866,
-      "rewards/margins": 0.042520731687545776,
-      "rewards/rejected": -0.23973730206489563,
-      "step": 640
-    },
-    {
-      "epoch": 0.3825779870512066,
-      "grad_norm": 55.28695481091037,
-      "learning_rate": 1.9128899352560328e-07,
-      "logits/chosen": -2.7750084400177,
-      "logits/rejected": -2.816450595855713,
-      "logps/chosen": -338.82080078125,
-      "logps/rejected": -358.87921142578125,
-      "loss": 0.6245,
-      "rewards/accuracies": 0.6666666269302368,
-      "rewards/chosen": -0.21482959389686584,
-      "rewards/margins": 0.2418464869260788,
-      "rewards/rejected": -0.45667609572410583,
-      "step": 650
-    },
-    {
-      "epoch": 0.38846380223660976,
-      "grad_norm": 60.1584611934493,
-      "learning_rate": 1.9423190111830488e-07,
-      "logits/chosen": -2.808283805847168,
-      "logits/rejected": -2.733508586883545,
-      "logps/chosen": -268.6055908203125,
-      "logps/rejected": -279.4296569824219,
-      "loss": 0.6244,
-      "rewards/accuracies": 0.5666666626930237,
-      "rewards/chosen": -0.24825648963451385,
-      "rewards/margins": 0.07473016530275345,
-      "rewards/rejected": -0.3229866623878479,
-      "step": 660
-    },
-    {
-      "epoch": 0.39434961742201297,
-      "grad_norm": 38.16884117276723,
-      "learning_rate": 1.9717480871100648e-07,
-      "logits/chosen": -2.763615131378174,
-      "logits/rejected": -2.742448329925537,
-      "logps/chosen": -214.5966796875,
-      "logps/rejected": -293.02313232421875,
-      "loss": 0.6017,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.1877942830324173,
-      "rewards/margins": 0.23820248246192932,
-      "rewards/rejected": -0.42599672079086304,
-      "step": 670
-    },
-    {
-      "epoch": 0.4002354326074161,
-      "grad_norm": 40.282667153165065,
-      "learning_rate": 2.0011771630370805e-07,
-      "logits/chosen": -2.7557873725891113,
-      "logits/rejected": -2.777015209197998,
-      "logps/chosen": -384.47186279296875,
-      "logps/rejected": -303.30877685546875,
-      "loss": 0.6045,
-      "rewards/accuracies": 0.7333332896232605,
-      "rewards/chosen": -0.15859545767307281,
-      "rewards/margins": 0.22654417157173157,
-      "rewards/rejected": -0.3851396441459656,
-      "step": 680
-    },
-    {
-      "epoch": 0.40612124779281933,
-      "grad_norm": 50.42522303402139,
-      "learning_rate": 2.0306062389640965e-07,
-      "logits/chosen": -2.7922425270080566,
-      "logits/rejected": -2.707688093185425,
-      "logps/chosen": -314.8849792480469,
-      "logps/rejected": -246.93618774414062,
-      "loss": 0.6023,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.20941126346588135,
-      "rewards/margins": 0.2189422845840454,
-      "rewards/rejected": -0.42835354804992676,
-      "step": 690
-    },
-    {
-      "epoch": 0.4120070629782225,
-      "grad_norm": 86.59616940333294,
-      "learning_rate": 2.0600353148911122e-07,
-      "logits/chosen": -2.7496159076690674,
-      "logits/rejected": -2.7615320682525635,
-      "logps/chosen": -316.3775939941406,
-      "logps/rejected": -316.27752685546875,
-      "loss": 0.6312,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.4660351276397705,
-      "rewards/margins": 0.19507920742034912,
-      "rewards/rejected": -0.6611142754554749,
-      "step": 700
-    },
-    {
-      "epoch": 0.41789287816362564,
-      "grad_norm": 113.88657488515956,
-      "learning_rate": 3.61272271440356e-07,
-      "logits/chosen": -2.713832139968872,
-      "logits/rejected": -2.639629364013672,
-      "logps/chosen": -292.7522277832031,
-      "logps/rejected": -276.9837951660156,
-      "loss": 0.6111,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.42747750878334045,
-      "rewards/margins": 0.30797505378723145,
-      "rewards/rejected": -0.7354525923728943,
-      "step": 710
-    },
-    {
-      "epoch": 0.42377869334902885,
-      "grad_norm": 58.276303041068935,
-      "learning_rate": 3.5664928281860625e-07,
-      "logits/chosen": -2.7681069374084473,
-      "logits/rejected": -2.6781115531921387,
-      "logps/chosen": -293.0718078613281,
-      "logps/rejected": -281.61212158203125,
-      "loss": 0.5922,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.3552438020706177,
-      "rewards/margins": 0.20946629345417023,
-      "rewards/rejected": -0.5647100806236267,
-      "step": 720
-    },
-    {
-      "epoch": 0.429664508534432,
-      "grad_norm": 34.26366533936239,
-      "learning_rate": 3.5198127192199e-07,
-      "logits/chosen": -2.7726988792419434,
-      "logits/rejected": -2.7980246543884277,
-      "logps/chosen": -358.5955810546875,
-      "logps/rejected": -417.20281982421875,
-      "loss": 0.5593,
-      "rewards/accuracies": 0.8333333134651184,
-      "rewards/chosen": -0.27696719765663147,
-      "rewards/margins": 0.5634738802909851,
-      "rewards/rejected": -0.8404410481452942,
-      "step": 730
-    },
-    {
-      "epoch": 0.4355503237198352,
-      "grad_norm": 48.29459945401339,
-      "learning_rate": 3.472702093635682e-07,
-      "logits/chosen": -2.822880268096924,
-      "logits/rejected": -2.754981279373169,
-      "logps/chosen": -232.97921752929688,
-      "logps/rejected": -225.82296752929688,
-      "loss": 0.6444,
-      "rewards/accuracies": 0.5333333015441895,
-      "rewards/chosen": -0.568128228187561,
-      "rewards/margins": -0.030178595334291458,
-      "rewards/rejected": -0.5379496216773987,
-      "step": 740
-    },
-    {
-      "epoch": 0.44143613890523836,
-      "grad_norm": 88.73737377845674,
-      "learning_rate": 3.42518083930774e-07,
-      "logits/chosen": -2.6345629692077637,
-      "logits/rejected": -2.6091387271881104,
-      "logps/chosen": -272.2128601074219,
-      "logps/rejected": -344.5741882324219,
-      "loss": 0.6375,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.46700435876846313,
-      "rewards/margins": 0.42694124579429626,
-      "rewards/rejected": -0.8939456939697266,
-      "step": 750
-    },
-    {
-      "epoch": 0.44732195409064157,
-      "grad_norm": 74.5070307164555,
-      "learning_rate": 3.377269017458409e-07,
-      "logits/chosen": -2.767118453979492,
-      "logits/rejected": -2.7667033672332764,
-      "logps/chosen": -392.59454345703125,
-      "logps/rejected": -410.0496520996094,
-      "loss": 0.5401,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.3810432553291321,
-      "rewards/margins": 0.6425568461418152,
-      "rewards/rejected": -1.0236002206802368,
-      "step": 760
-    },
-    {
-      "epoch": 0.4532077692760447,
-      "grad_norm": 102.79976551674226,
-      "learning_rate": 3.3289868541891255e-07,
-      "logits/chosen": -2.604182720184326,
-      "logits/rejected": -2.527705430984497,
-      "logps/chosen": -296.3623046875,
-      "logps/rejected": -354.95819091796875,
-      "loss": 0.6152,
-      "rewards/accuracies": 0.6666666865348816,
-      "rewards/chosen": -0.5251516103744507,
-      "rewards/margins": 0.46194180846214294,
-      "rewards/rejected": -0.9870933294296265,
-      "step": 770
-    },
-    {
-      "epoch": 0.45909358446144793,
-      "grad_norm": 67.73780235887719,
-      "learning_rate": 3.280354731941932e-07,
-      "logits/chosen": -2.76226544380188,
-      "logits/rejected": -2.6936070919036865,
-      "logps/chosen": -315.97216796875,
-      "logps/rejected": -342.60546875,
-      "loss": 0.5639,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.4423181116580963,
-      "rewards/margins": 0.47899603843688965,
-      "rewards/rejected": -0.9213140606880188,
-      "step": 780
-    },
-    {
-      "epoch": 0.4649793996468511,
-      "grad_norm": 38.93430396007463,
-      "learning_rate": 3.231393180894973e-07,
-      "logits/chosen": -2.795037031173706,
-      "logits/rejected": -2.7878684997558594,
-      "logps/chosen": -255.4384765625,
-      "logps/rejected": -313.3453369140625,
-      "loss": 0.5453,
-      "rewards/accuracies": 0.7333332896232605,
-      "rewards/chosen": -0.3891940712928772,
-      "rewards/margins": 0.5739094614982605,
-      "rewards/rejected": -0.9631035923957825,
-      "step": 790
-    },
-    {
-      "epoch": 0.47086521483225424,
-      "grad_norm": 78.21647434951635,
-      "learning_rate": 3.1821228702956253e-07,
-      "logits/chosen": -2.5857443809509277,
-      "logits/rejected": -2.5272181034088135,
-      "logps/chosen": -269.7416076660156,
-      "logps/rejected": -359.6760559082031,
-      "loss": 0.6535,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.6391388773918152,
-      "rewards/margins": 0.2941351532936096,
-      "rewards/rejected": -0.9332740902900696,
-      "step": 800
-    },
-    {
-      "epoch": 0.47675103001765745,
-      "grad_norm": 59.73596890978745,
-      "learning_rate": 3.132564599734921e-07,
-      "logits/chosen": -2.7386457920074463,
-      "logits/rejected": -2.754356861114502,
-      "logps/chosen": -292.2206726074219,
-      "logps/rejected": -324.42816162109375,
-      "loss": 0.6368,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -0.3809540867805481,
-      "rewards/margins": 0.2144467830657959,
-      "rewards/rejected": -0.5954009294509888,
-      "step": 810
-    },
-    {
-      "epoch": 0.4826368452030606,
-      "grad_norm": 58.04453100186318,
-      "learning_rate": 3.082739290366946e-07,
-      "logits/chosen": -2.731048107147217,
-      "logits/rejected": -2.7188940048217773,
-      "logps/chosen": -367.1738586425781,
-      "logps/rejected": -365.79803466796875,
-      "loss": 0.5855,
-      "rewards/accuracies": 0.7333333492279053,
-      "rewards/chosen": -0.40680187940597534,
-      "rewards/margins": 0.5839999318122864,
-      "rewards/rejected": -0.9908016920089722,
-      "step": 820
-    },
-    {
-      "epoch": 0.4885226603884638,
-      "grad_norm": 60.703085038442914,
-      "learning_rate": 3.0326679760769226e-07,
-      "logits/chosen": -2.633038282394409,
-      "logits/rejected": -2.6379826068878174,
-      "logps/chosen": -270.77972412109375,
-      "logps/rejected": -361.93170166015625,
-      "loss": 0.6076,
-      "rewards/accuracies": 0.8333333730697632,
-      "rewards/chosen": -0.29258260130882263,
-      "rewards/margins": 0.578423798084259,
-      "rewards/rejected": -0.8710063695907593,
-      "step": 830
-    },
-    {
-      "epoch": 0.49440847557386697,
-      "grad_norm": 68.79606387966484,
-      "learning_rate": 2.982371794601694e-07,
-      "logits/chosen": -2.7220444679260254,
-      "logits/rejected": -2.665432929992676,
-      "logps/chosen": -456.65557861328125,
-      "logps/rejected": -338.92926025390625,
-      "loss": 0.6101,
-      "rewards/accuracies": 0.7333333492279053,
-      "rewards/chosen": -0.38200992345809937,
-      "rewards/margins": 0.4571179747581482,
-      "rewards/rejected": -0.8391278386116028,
-      "step": 840
-    },
-    {
-      "epoch": 0.5002942907592701,
-      "grad_norm": 39.349812488010386,
-      "learning_rate": 2.931871978606379e-07,
-      "logits/chosen": -2.7449920177459717,
-      "logits/rejected": -2.785231590270996,
-      "logps/chosen": -353.2920837402344,
-      "logps/rejected": -434.22979736328125,
-      "loss": 0.5017,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.3858256936073303,
-      "rewards/margins": 0.6933655738830566,
-      "rewards/rejected": -1.0791910886764526,
-      "step": 850
-    },
-    {
-      "epoch": 0.5061801059446733,
-      "grad_norm": 65.24381800745265,
-      "learning_rate": 2.881189846720941e-07,
-      "logits/chosen": -2.7466366291046143,
-      "logits/rejected": -2.7046473026275635,
-      "logps/chosen": -334.4920959472656,
-      "logps/rejected": -363.40838623046875,
-      "loss": 0.5525,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.41190671920776367,
-      "rewards/margins": 0.559307873249054,
-      "rewards/rejected": -0.9712145924568176,
-      "step": 860
-    },
-    {
-      "epoch": 0.5120659211300765,
-      "grad_norm": 51.09800858702452,
-      "learning_rate": 2.830346794540479e-07,
-      "logits/chosen": -2.6742959022521973,
-      "logits/rejected": -2.6640892028808594,
-      "logps/chosen": -266.5992736816406,
-      "logps/rejected": -313.98828125,
-      "loss": 0.5778,
-      "rewards/accuracies": 0.4999999403953552,
-      "rewards/chosen": -0.6418452262878418,
-      "rewards/margins": 0.16775313019752502,
-      "rewards/rejected": -0.8095983266830444,
-      "step": 870
-    },
-    {
-      "epoch": 0.5179517363154797,
-      "grad_norm": 54.983291652847775,
-      "learning_rate": 2.7793642855930133e-07,
-      "logits/chosen": -2.7972800731658936,
-      "logits/rejected": -2.657902240753174,
-      "logps/chosen": -379.5608825683594,
-      "logps/rejected": -415.2431640625,
-      "loss": 0.6336,
-      "rewards/accuracies": 0.76666659116745,
-      "rewards/chosen": -0.5884305834770203,
-      "rewards/margins": 0.4619186818599701,
-      "rewards/rejected": -1.0503493547439575,
-      "step": 880
-    },
-    {
-      "epoch": 0.5238375515008828,
-      "grad_norm": 52.1793743861904,
-      "learning_rate": 2.728263842278608e-07,
-      "logits/chosen": -2.7541468143463135,
-      "logits/rejected": -2.7553248405456543,
-      "logps/chosen": -340.1725158691406,
-      "logps/rejected": -431.4141540527344,
-      "loss": 0.5083,
-      "rewards/accuracies": 0.8333333134651184,
-      "rewards/chosen": -0.5956707000732422,
-      "rewards/margins": 0.6665983200073242,
-      "rewards/rejected": -1.2622690200805664,
-      "step": 890
-    },
-    {
-      "epoch": 0.5297233666862861,
-      "grad_norm": 62.43287153719138,
-      "learning_rate": 2.6770670367836276e-07,
-      "logits/chosen": -2.731095790863037,
-      "logits/rejected": -2.734687328338623,
-      "logps/chosen": -333.5868835449219,
-      "logps/rejected": -405.0290222167969,
-      "loss": 0.5372,
-      "rewards/accuracies": 0.7666667103767395,
-      "rewards/chosen": -0.7237066030502319,
-      "rewards/margins": 0.5302045941352844,
-      "rewards/rejected": -1.2539112567901611,
-      "step": 900
-    },
-    {
-      "epoch": 0.5356091818716893,
-      "grad_norm": 82.01285085767707,
-      "learning_rate": 2.625795481973985e-07,
-      "logits/chosen": -2.6702892780303955,
-      "logits/rejected": -2.5133378505706787,
-      "logps/chosen": -338.619873046875,
-      "logps/rejected": -311.3938903808594,
-      "loss": 0.5897,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -0.5718550086021423,
-      "rewards/margins": 0.4375828206539154,
-      "rewards/rejected": -1.0094377994537354,
-      "step": 910
-    },
-    {
-      "epoch": 0.5414949970570924,
-      "grad_norm": 95.54109435400643,
-      "learning_rate": 2.5744708222712137e-07,
-      "logits/chosen": -2.7311172485351562,
-      "logits/rejected": -2.739924907684326,
-      "logps/chosen": -272.01995849609375,
-      "logps/rejected": -311.47833251953125,
-      "loss": 0.5762,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.5505859851837158,
-      "rewards/margins": 0.6296683549880981,
-      "rewards/rejected": -1.180254340171814,
-      "step": 920
-    },
-    {
-      "epoch": 0.5473808122424956,
-      "grad_norm": 36.48587725082776,
-      "learning_rate": 2.5231147245152156e-07,
-      "logits/chosen": -2.4503333568573,
-      "logits/rejected": -2.468564033508301,
-      "logps/chosen": -298.019775390625,
-      "logps/rejected": -372.83026123046875,
-      "loss": 0.5587,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.6832865476608276,
-      "rewards/margins": 0.8449433445930481,
-      "rewards/rejected": -1.5282299518585205,
-      "step": 930
-    },
-    {
-      "epoch": 0.5532666274278988,
-      "grad_norm": 71.24789616161829,
-      "learning_rate": 2.471748868817551e-07,
-      "logits/chosen": -2.7247791290283203,
-      "logits/rejected": -2.7448885440826416,
-      "logps/chosen": -316.38726806640625,
-      "logps/rejected": -418.75830078125,
-      "loss": 0.5429,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": -0.43857964873313904,
-      "rewards/margins": 0.8119268417358398,
-      "rewards/rejected": -1.2505066394805908,
-      "step": 940
-    },
-    {
-      "epoch": 0.559152442613302,
-      "grad_norm": 57.37226759319327,
-      "learning_rate": 2.4203949394091186e-07,
-      "logits/chosen": -2.7133288383483887,
-      "logits/rejected": -2.6669368743896484,
-      "logps/chosen": -384.8581848144531,
-      "logps/rejected": -477.08538818359375,
-      "loss": 0.5152,
-      "rewards/accuracies": 0.7999999523162842,
-      "rewards/chosen": -0.9175896644592285,
-      "rewards/margins": 1.0064847469329834,
-      "rewards/rejected": -1.9240741729736328,
-      "step": 950
-    },
-    {
-      "epoch": 0.5650382577987051,
-      "grad_norm": 102.83287628895852,
-      "learning_rate": 2.3690746154861048e-07,
-      "logits/chosen": -2.5697152614593506,
-      "logits/rejected": -2.5663580894470215,
-      "logps/chosen": -361.67926025390625,
-      "logps/rejected": -425.91571044921875,
-      "loss": 0.5645,
-      "rewards/accuracies": 0.7333333492279053,
-      "rewards/chosen": -0.9139325022697449,
-      "rewards/margins": 0.5984528064727783,
-      "rewards/rejected": -1.5123852491378784,
-      "step": 960
-    },
-    {
-      "epoch": 0.5709240729841083,
-      "grad_norm": 103.97188710606014,
-      "learning_rate": 2.3178095620580524e-07,
-      "logits/chosen": -2.642576217651367,
-      "logits/rejected": -2.576850175857544,
-      "logps/chosen": -394.79290771484375,
-      "logps/rejected": -357.9416809082031,
-      "loss": 0.628,
-      "rewards/accuracies": 0.5999999642372131,
-      "rewards/chosen": -1.2030242681503296,
-      "rewards/margins": 0.2695932388305664,
-      "rewards/rejected": -1.4726173877716064,
-      "step": 970
-    },
-    {
-      "epoch": 0.5768098881695115,
-      "grad_norm": 122.23429828784404,
-      "learning_rate": 2.266621420801918e-07,
-      "logits/chosen": -2.3418798446655273,
-      "logits/rejected": -2.3062217235565186,
-      "logps/chosen": -334.18182373046875,
-      "logps/rejected": -375.04083251953125,
-      "loss": 0.5985,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.9751673936843872,
-      "rewards/margins": 0.36949509382247925,
-      "rewards/rejected": -1.3446625471115112,
-      "step": 980
-    },
-    {
-      "epoch": 0.5826957033549146,
-      "grad_norm": 53.90742131111234,
-      "learning_rate": 2.2155318009259821e-07,
-      "logits/chosen": -2.517277479171753,
-      "logits/rejected": -2.552461862564087,
-      "logps/chosen": -351.4830322265625,
-      "logps/rejected": -347.33770751953125,
-      "loss": 0.6017,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.7861822843551636,
-      "rewards/margins": 0.5186395645141602,
-      "rewards/rejected": -1.3048217296600342,
-      "step": 990
-    },
-    {
-      "epoch": 0.5885815185403178,
-      "grad_norm": 79.56155835501555,
-      "learning_rate": 2.1645622700474695e-07,
-      "logits/chosen": -2.53051495552063,
-      "logits/rejected": -2.50981068611145,
-      "logps/chosen": -300.99261474609375,
-      "logps/rejected": -307.985107421875,
-      "loss": 0.6313,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.9508684873580933,
-      "rewards/margins": 0.4335688054561615,
-      "rewards/rejected": -1.3844374418258667,
-      "step": 1000
-    },
-    {
-      "epoch": 0.594467333725721,
-      "grad_norm": 178.75847915531864,
-      "learning_rate": 2.113734345087716e-07,
-      "logits/chosen": -2.699451208114624,
-      "logits/rejected": -2.7512571811676025,
-      "logps/chosen": -213.2908172607422,
-      "logps/rejected": -347.00457763671875,
-      "loss": 0.4959,
-      "rewards/accuracies": 0.6666666865348816,
-      "rewards/chosen": -0.5839646458625793,
-      "rewards/margins": 0.5955744385719299,
-      "rewards/rejected": -1.1795389652252197,
-      "step": 1010
-    },
-    {
-      "epoch": 0.6003531489111242,
-      "grad_norm": 144.9877689912697,
-      "learning_rate": 2.0630694831887473e-07,
-      "logits/chosen": -2.3605151176452637,
-      "logits/rejected": -2.377347469329834,
-      "logps/chosen": -302.31072998046875,
-      "logps/rejected": -364.3263854980469,
-      "loss": 0.5393,
-      "rewards/accuracies": 0.7333333492279053,
-      "rewards/chosen": -0.8714925646781921,
-      "rewards/margins": 0.615392804145813,
-      "rewards/rejected": -1.4868853092193604,
-      "step": 1020
-    },
-    {
-      "epoch": 0.6062389640965273,
-      "grad_norm": 90.93107910277243,
-      "learning_rate": 2.0125890726550896e-07,
-      "logits/chosen": -2.774040937423706,
-      "logits/rejected": -2.749173641204834,
-      "logps/chosen": -312.368896484375,
-      "logps/rejected": -329.6822814941406,
-      "loss": 0.5044,
-      "rewards/accuracies": 0.8333333730697632,
-      "rewards/chosen": -0.3795444369316101,
-      "rewards/margins": 0.5973297357559204,
-      "rewards/rejected": -0.9768741726875305,
-      "step": 1030
-    },
-    {
-      "epoch": 0.6121247792819305,
-      "grad_norm": 53.759522144352374,
-      "learning_rate": 1.9623144239246432e-07,
-      "logits/chosen": -2.555093765258789,
-      "logits/rejected": -2.572145700454712,
-      "logps/chosen": -320.54248046875,
-      "logps/rejected": -383.291748046875,
-      "loss": 0.5761,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.7120213508605957,
-      "rewards/margins": 0.4912422299385071,
-      "rewards/rejected": -1.203263521194458,
-      "step": 1040
-    },
-    {
-      "epoch": 0.6180105944673338,
-      "grad_norm": 96.22956917547782,
-      "learning_rate": 1.91226676057242e-07,
-      "logits/chosen": -2.5407838821411133,
-      "logits/rejected": -2.471928596496582,
-      "logps/chosen": -419.9786071777344,
-      "logps/rejected": -386.1612854003906,
-      "loss": 0.5582,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.8196290135383606,
-      "rewards/margins": 0.6197575330734253,
-      "rewards/rejected": -1.4393866062164307,
-      "step": 1050
-    },
-    {
-      "epoch": 0.6238964096527368,
-      "grad_norm": 84.05938732777776,
-      "learning_rate": 1.8624672103509578e-07,
-      "logits/chosen": -2.4780263900756836,
-      "logits/rejected": -2.3386425971984863,
-      "logps/chosen": -417.42724609375,
-      "logps/rejected": -414.38262939453125,
-      "loss": 0.5922,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.0109527111053467,
-      "rewards/margins": 0.40876656770706177,
-      "rewards/rejected": -1.4197193384170532,
-      "step": 1060
-    },
-    {
-      "epoch": 0.6297822248381401,
-      "grad_norm": 125.25073964918506,
-      "learning_rate": 1.8129367962711882e-07,
-      "logits/chosen": -2.6754302978515625,
-      "logits/rejected": -2.6922547817230225,
-      "logps/chosen": -416.434326171875,
-      "logps/rejected": -419.4442443847656,
-      "loss": 0.5897,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.8944675326347351,
-      "rewards/margins": 0.5295539498329163,
-      "rewards/rejected": -1.4240214824676514,
-      "step": 1070
-    },
-    {
-      "epoch": 0.6356680400235433,
-      "grad_norm": 87.45258473406294,
-      "learning_rate": 1.7636964277275108e-07,
-      "logits/chosen": -2.5234591960906982,
-      "logits/rejected": -2.5611681938171387,
-      "logps/chosen": -326.291015625,
-      "logps/rejected": -389.78839111328125,
-      "loss": 0.4929,
-      "rewards/accuracies": 0.7999999523162842,
-      "rewards/chosen": -0.7259222269058228,
-      "rewards/margins": 0.8374055624008179,
-      "rewards/rejected": -1.5633279085159302,
-      "step": 1080
-    },
-    {
-      "epoch": 0.6415538552089465,
-      "grad_norm": 53.279206385221514,
-      "learning_rate": 1.7147668916708398e-07,
-      "logits/chosen": -2.480179786682129,
-      "logits/rejected": -2.4370687007904053,
-      "logps/chosen": -358.24444580078125,
-      "logps/rejected": -384.9347229003906,
-      "loss": 0.5857,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -1.065148115158081,
-      "rewards/margins": 0.38636088371276855,
-      "rewards/rejected": -1.4515092372894287,
-      "step": 1090
-    },
-    {
-      "epoch": 0.6474396703943496,
-      "grad_norm": 52.091641562005016,
-      "learning_rate": 1.666168843833339e-07,
-      "logits/chosen": -2.3647563457489014,
-      "logits/rejected": -2.3754029273986816,
-      "logps/chosen": -263.02447509765625,
-      "logps/rejected": -354.109619140625,
-      "loss": 0.5574,
-      "rewards/accuracies": 0.8333333134651184,
-      "rewards/chosen": -0.5551760792732239,
-      "rewards/margins": 0.844565749168396,
-      "rewards/rejected": -1.399741768836975,
-      "step": 1100
-    },
-    {
-      "epoch": 0.6533254855797528,
-      "grad_norm": 112.76354894611299,
-      "learning_rate": 1.6179228000085532e-07,
-      "logits/chosen": -2.617955446243286,
-      "logits/rejected": -2.5767765045166016,
-      "logps/chosen": -301.1888122558594,
-      "logps/rejected": -372.39239501953125,
-      "loss": 0.6259,
-      "rewards/accuracies": 0.6666666269302368,
-      "rewards/chosen": -0.8070073127746582,
-      "rewards/margins": 0.5521813035011292,
-      "rewards/rejected": -1.3591885566711426,
-      "step": 1110
-    },
-    {
-      "epoch": 0.659211300765156,
-      "grad_norm": 72.36752468174844,
-      "learning_rate": 1.570049127390604e-07,
-      "logits/chosen": -2.607243299484253,
-      "logits/rejected": -2.6284470558166504,
-      "logps/chosen": -382.6817321777344,
-      "logps/rejected": -473.088134765625,
-      "loss": 0.5981,
-      "rewards/accuracies": 0.5999999642372131,
-      "rewards/chosen": -0.9492104649543762,
-      "rewards/margins": 0.6012105941772461,
-      "rewards/rejected": -1.5504209995269775,
-      "step": 1120
-    },
-    {
-      "epoch": 0.6650971159505592,
-      "grad_norm": 99.38680104764003,
-      "learning_rate": 1.5225680359761325e-07,
-      "logits/chosen": -2.555722713470459,
-      "logits/rejected": -2.5732975006103516,
-      "logps/chosen": -235.4317169189453,
-      "logps/rejected": -325.0386962890625,
-      "loss": 0.5918,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.6323750019073486,
-      "rewards/margins": 0.4906527101993561,
-      "rewards/rejected": -1.1230276823043823,
-      "step": 1130
-    },
-    {
-      "epoch": 0.6709829311359623,
-      "grad_norm": 113.18996714616782,
-      "learning_rate": 1.4754995700325917e-07,
-      "logits/chosen": -2.628831624984741,
-      "logits/rejected": -2.61437726020813,
-      "logps/chosen": -304.4407043457031,
-      "logps/rejected": -390.73321533203125,
-      "loss": 0.5161,
-      "rewards/accuracies": 0.7666667103767395,
-      "rewards/chosen": -0.7466219067573547,
-      "rewards/margins": 0.5649534463882446,
-      "rewards/rejected": -1.3115752935409546,
-      "step": 1140
-    },
-    {
-      "epoch": 0.6768687463213655,
-      "grad_norm": 213.32924667526723,
-      "learning_rate": 1.428863599636511e-07,
-      "logits/chosen": -2.4004693031311035,
-      "logits/rejected": -2.5256259441375732,
-      "logps/chosen": -326.31280517578125,
-      "logps/rejected": -392.92529296875,
-      "loss": 0.4886,
-      "rewards/accuracies": 0.73333340883255,
-      "rewards/chosen": -0.8898285627365112,
-      "rewards/margins": 0.6658565998077393,
-      "rewards/rejected": -1.5556851625442505,
-      "step": 1150
-    },
-    {
-      "epoch": 0.6827545615067687,
-      "grad_norm": 169.10265364288662,
-      "learning_rate": 1.3826798122852868e-07,
-      "logits/chosen": -2.6862120628356934,
-      "logits/rejected": -2.5977320671081543,
-      "logps/chosen": -360.7554016113281,
-      "logps/rejected": -339.32513427734375,
-      "loss": 0.5558,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.9063922166824341,
-      "rewards/margins": 0.3310279846191406,
-      "rewards/rejected": -1.2374203205108643,
-      "step": 1160
-    },
-    {
-      "epoch": 0.6886403766921718,
-      "grad_norm": 173.65784239229404,
-      "learning_rate": 1.3369677045860544e-07,
-      "logits/chosen": -2.625609874725342,
-      "logits/rejected": -2.5897438526153564,
-      "logps/chosen": -319.73748779296875,
-      "logps/rejected": -373.0987854003906,
-      "loss": 0.4981,
-      "rewards/accuracies": 0.8333333730697632,
-      "rewards/chosen": -0.6690064668655396,
-      "rewards/margins": 0.8604344129562378,
-      "rewards/rejected": -1.5294408798217773,
-      "step": 1170
-    },
-    {
-      "epoch": 0.694526191877575,
-      "grad_norm": 124.29493358693395,
-      "learning_rate": 1.291746574025148e-07,
-      "logits/chosen": -2.736769914627075,
-      "logits/rejected": -2.7117438316345215,
-      "logps/chosen": -440.48779296875,
-      "logps/rejected": -379.79132080078125,
-      "loss": 0.5748,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -1.1783974170684814,
-      "rewards/margins": 0.2958952486515045,
-      "rewards/rejected": -1.4742927551269531,
-      "step": 1180
-    },
-    {
-      "epoch": 0.7004120070629782,
-      "grad_norm": 78.3006038615153,
-      "learning_rate": 1.247035510821611e-07,
-      "logits/chosen": -2.7176737785339355,
-      "logits/rejected": -2.6454663276672363,
-      "logps/chosen": -376.97381591796875,
-      "logps/rejected": -335.2852783203125,
-      "loss": 0.5881,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.9591771960258484,
-      "rewards/margins": 0.3564377725124359,
-      "rewards/rejected": -1.315614938735962,
-      "step": 1190
-    },
-    {
-      "epoch": 0.7062978222483814,
-      "grad_norm": 173.0463100342363,
-      "learning_rate": 1.2028533898682123e-07,
-      "logits/chosen": -2.7012906074523926,
-      "logits/rejected": -2.6718831062316895,
-      "logps/chosen": -352.58294677734375,
-      "logps/rejected": -393.17572021484375,
-      "loss": 0.5845,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.8266950845718384,
-      "rewards/margins": 0.6615980863571167,
-      "rewards/rejected": -1.4882930517196655,
-      "step": 1200
-    },
-    {
-      "epoch": 0.7121836374337845,
-      "grad_norm": 80.48548009967331,
-      "learning_rate": 1.1592188627633598e-07,
-      "logits/chosen": -2.583117961883545,
-      "logits/rejected": -2.6228084564208984,
-      "logps/chosen": -317.26708984375,
-      "logps/rejected": -329.43719482421875,
-      "loss": 0.632,
-      "rewards/accuracies": 0.7333332896232605,
-      "rewards/chosen": -0.8232676386833191,
-      "rewards/margins": 0.3029004633426666,
-      "rewards/rejected": -1.1261680126190186,
-      "step": 1210
-    },
-    {
-      "epoch": 0.7180694526191878,
-      "grad_norm": 62.307227511632156,
-      "learning_rate": 1.1161503499372835e-07,
-      "logits/chosen": -2.5381112098693848,
-      "logits/rejected": -2.426206350326538,
-      "logps/chosen": -356.4806823730469,
-      "logps/rejected": -495.2960510253906,
-      "loss": 0.4735,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.9427182078361511,
-      "rewards/margins": 0.6752321124076843,
-      "rewards/rejected": -1.617950439453125,
-      "step": 1220
-    },
-    {
-      "epoch": 0.723955267804591,
-      "grad_norm": 102.88342581327655,
-      "learning_rate": 1.0736660328757954e-07,
-      "logits/chosen": -2.63267183303833,
-      "logits/rejected": -2.4871132373809814,
-      "logps/chosen": -303.36163330078125,
-      "logps/rejected": -332.2799987792969,
-      "loss": 0.5103,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.6926143765449524,
-      "rewards/margins": 0.48562532663345337,
-      "rewards/rejected": -1.1782395839691162,
-      "step": 1230
-    },
-    {
-      "epoch": 0.7298410829899941,
-      "grad_norm": 69.73162807423068,
-      "learning_rate": 1.0317838464449318e-07,
-      "logits/chosen": -2.5889840126037598,
-      "logits/rejected": -2.594730854034424,
-      "logps/chosen": -305.60546875,
-      "logps/rejected": -347.56988525390625,
-      "loss": 0.5767,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.6716678142547607,
-      "rewards/margins": 0.4861713945865631,
-      "rewards/rejected": -1.1578391790390015,
-      "step": 1240
-    },
-    {
-      "epoch": 0.7357268981753973,
-      "grad_norm": 99.72345975810403,
-      "learning_rate": 9.905214713197019e-08,
-      "logits/chosen": -2.780867338180542,
-      "logits/rejected": -2.7212603092193604,
-      "logps/chosen": -319.85809326171875,
-      "logps/rejected": -362.9429931640625,
-      "loss": 0.5763,
-      "rewards/accuracies": 0.7333333492279053,
-      "rewards/chosen": -0.7506597638130188,
-      "rewards/margins": 0.7295576333999634,
-      "rewards/rejected": -1.480217456817627,
-      "step": 1250
-    },
-    {
-      "epoch": 0.7416127133608005,
-      "grad_norm": 42.256039163587936,
-      "learning_rate": 9.498963265201454e-08,
-      "logits/chosen": -2.5502285957336426,
-      "logits/rejected": -2.4693973064422607,
-      "logps/chosen": -301.47845458984375,
-      "logps/rejected": -348.4825439453125,
-      "loss": 0.5623,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -0.7193578481674194,
-      "rewards/margins": 0.40803733468055725,
-      "rewards/rejected": -1.1273951530456543,
-      "step": 1260
-    },
-    {
-      "epoch": 0.7474985285462037,
-      "grad_norm": 51.960392917513985,
-      "learning_rate": 9.09925562057845e-08,
-      "logits/chosen": -2.5758213996887207,
-      "logits/rejected": -2.6676766872406006,
-      "logps/chosen": -319.75018310546875,
-      "logps/rejected": -392.013916015625,
-      "loss": 0.5471,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -0.8297551870346069,
-      "rewards/margins": 0.583861231803894,
-      "rewards/rejected": -1.4136162996292114,
-      "step": 1270
-    },
-    {
-      "epoch": 0.7533843437316068,
-      "grad_norm": 96.3763472702637,
-      "learning_rate": 8.706260516960043e-08,
-      "logits/chosen": -2.6656923294067383,
-      "logits/rejected": -2.706164836883545,
-      "logps/chosen": -342.38623046875,
-      "logps/rejected": -424.30706787109375,
-      "loss": 0.6335,
-      "rewards/accuracies": 0.6666666865348816,
-      "rewards/chosen": -1.1196720600128174,
-      "rewards/margins": 0.4281339645385742,
-      "rewards/rejected": -1.5478060245513916,
-      "step": 1280
-    },
-    {
-      "epoch": 0.75927015891701,
-      "grad_norm": 82.6639251391733,
-      "learning_rate": 8.320143858261486e-08,
-      "logits/chosen": -2.746375560760498,
-      "logits/rejected": -2.6994574069976807,
-      "logps/chosen": -360.11285400390625,
-      "logps/rejected": -339.80926513671875,
-      "loss": 0.5403,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -0.801996111869812,
-      "rewards/margins": 0.3962213695049286,
-      "rewards/rejected": -1.1982173919677734,
-      "step": 1290
-    },
-    {
-      "epoch": 0.7651559741024132,
-      "grad_norm": 58.83037988316151,
-      "learning_rate": 7.941068644644405e-08,
-      "logits/chosen": -2.7802703380584717,
-      "logits/rejected": -2.643925428390503,
-      "logps/chosen": -391.9327697753906,
-      "logps/rejected": -413.474609375,
-      "loss": 0.5632,
-      "rewards/accuracies": 0.7333333492279053,
-      "rewards/chosen": -1.1195310354232788,
-      "rewards/margins": 0.615394115447998,
-      "rewards/rejected": -1.7349250316619873,
-      "step": 1300
-    },
-    {
-      "epoch": 0.7710417892878164,
-      "grad_norm": 73.64616872899245,
-      "learning_rate": 7.569194903705913e-08,
-      "logits/chosen": -2.7224087715148926,
-      "logits/rejected": -2.751784324645996,
-      "logps/chosen": -406.634521484375,
-      "logps/rejected": -462.2149963378906,
-      "loss": 0.5761,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -0.9177020788192749,
-      "rewards/margins": 0.5259424448013306,
-      "rewards/rejected": -1.4436447620391846,
-      "step": 1310
-    },
-    {
-      "epoch": 0.7769276044732195,
-      "grad_norm": 226.35543077516525,
-      "learning_rate": 7.204679622922502e-08,
-      "logits/chosen": -2.6812875270843506,
-      "logits/rejected": -2.663227081298828,
-      "logps/chosen": -432.437744140625,
-      "logps/rejected": -406.0752258300781,
-      "loss": 0.6171,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.8880939483642578,
-      "rewards/margins": 0.6238939166069031,
-      "rewards/rejected": -1.5119879245758057,
-      "step": 1320
-    },
-    {
-      "epoch": 0.7828134196586227,
-      "grad_norm": 52.032206635043295,
-      "learning_rate": 6.847676683377398e-08,
-      "logits/chosen": -2.6559484004974365,
-      "logits/rejected": -2.5057730674743652,
-      "logps/chosen": -473.78338623046875,
-      "logps/rejected": -408.1139221191406,
-      "loss": 0.5067,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.8587116003036499,
-      "rewards/margins": 0.6869573593139648,
-      "rewards/rejected": -1.5456688404083252,
-      "step": 1330
-    },
-    {
-      "epoch": 0.7886992348440259,
-      "grad_norm": 313.43589076362645,
-      "learning_rate": 6.498336794799234e-08,
-      "logits/chosen": -2.5412328243255615,
-      "logits/rejected": -2.4910500049591064,
-      "logps/chosen": -325.08929443359375,
-      "logps/rejected": -471.27276611328125,
-      "loss": 0.5095,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.8783607482910156,
-      "rewards/margins": 0.6811479926109314,
-      "rewards/rejected": -1.5595088005065918,
-      "step": 1340
-    },
-    {
-      "epoch": 0.794585050029429,
-      "grad_norm": 89.6143819430957,
-      "learning_rate": 6.156807431939537e-08,
-      "logits/chosen": -2.522613763809204,
-      "logits/rejected": -2.5427727699279785,
-      "logps/chosen": -385.08453369140625,
-      "logps/rejected": -421.5887756347656,
-      "loss": 0.5045,
-      "rewards/accuracies": 0.8333333730697632,
-      "rewards/chosen": -0.6811944246292114,
-      "rewards/margins": 0.6432855725288391,
-      "rewards/rejected": -1.3244801759719849,
-      "step": 1350
-    },
-    {
-      "epoch": 0.8004708652148322,
-      "grad_norm": 67.09755437948724,
-      "learning_rate": 5.8232327723159e-08,
-      "logits/chosen": -2.4834189414978027,
-      "logits/rejected": -2.527981996536255,
-      "logps/chosen": -332.33831787109375,
-      "logps/rejected": -436.96856689453125,
-      "loss": 0.5009,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.9011951684951782,
-      "rewards/margins": 1.019168496131897,
-      "rewards/rejected": -1.920363426208496,
-      "step": 1360
-    },
-    {
-      "epoch": 0.8063566804002354,
-      "grad_norm": 58.01149580333495,
-      "learning_rate": 5.49775363534706e-08,
-      "logits/chosen": -2.6389904022216797,
-      "logits/rejected": -2.5821375846862793,
-      "logps/chosen": -338.70745849609375,
-      "logps/rejected": -419.2255859375,
-      "loss": 0.4777,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.6776565909385681,
-      "rewards/margins": 0.8060138821601868,
-      "rewards/rejected": -1.4836704730987549,
-      "step": 1370
-    },
-    {
-      "epoch": 0.8122424955856387,
-      "grad_norm": 102.8449817133206,
-      "learning_rate": 5.180507422905583e-08,
-      "logits/chosen": -2.7128119468688965,
-      "logits/rejected": -2.673996925354004,
-      "logps/chosen": -322.67779541015625,
-      "logps/rejected": -363.71270751953125,
-      "loss": 0.5261,
-      "rewards/accuracies": 0.7999999523162842,
-      "rewards/chosen": -0.8014975786209106,
-      "rewards/margins": 0.7609502673149109,
-      "rewards/rejected": -1.5624477863311768,
-      "step": 1380
-    },
-    {
-      "epoch": 0.8181283107710418,
-      "grad_norm": 65.36829776160975,
-      "learning_rate": 4.871628061313321e-08,
-      "logits/chosen": -2.581416368484497,
-      "logits/rejected": -2.682572364807129,
-      "logps/chosen": -318.94915771484375,
-      "logps/rejected": -439.8599548339844,
-      "loss": 0.5081,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": -0.6443731188774109,
-      "rewards/margins": 1.2208740711212158,
-      "rewards/rejected": -1.8652470111846924,
-      "step": 1390
-    },
-    {
-      "epoch": 0.824014125956445,
-      "grad_norm": 178.13037283488754,
-      "learning_rate": 4.571245944804089e-08,
-      "logits/chosen": -2.489166736602783,
-      "logits/rejected": -2.398707866668701,
-      "logps/chosen": -375.8815612792969,
-      "logps/rejected": -497.11761474609375,
-      "loss": 0.5097,
-      "rewards/accuracies": 0.8333333134651184,
-      "rewards/chosen": -0.9333661198616028,
-      "rewards/margins": 1.0528299808502197,
-      "rewards/rejected": -1.9861959218978882,
-      "step": 1400
-    },
-    {
-      "epoch": 0.8298999411418482,
-      "grad_norm": 172.4847999858577,
-      "learning_rate": 4.279487880477342e-08,
-      "logits/chosen": -2.5642523765563965,
-      "logits/rejected": -2.607612133026123,
-      "logps/chosen": -372.1091003417969,
-      "logps/rejected": -393.6594543457031,
-      "loss": 0.5695,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -0.8643810153007507,
-      "rewards/margins": 0.6560724973678589,
-      "rewards/rejected": -1.5204535722732544,
-      "step": 1410
-    },
-    {
-      "epoch": 0.8357857563272513,
-      "grad_norm": 88.7462733661287,
-      "learning_rate": 3.9964770347662715e-08,
-      "logits/chosen": -2.615901470184326,
-      "logits/rejected": -2.5380148887634277,
-      "logps/chosen": -269.59002685546875,
-      "logps/rejected": -408.44708251953125,
-      "loss": 0.5431,
-      "rewards/accuracies": 0.6666666865348816,
-      "rewards/chosen": -0.8203733563423157,
-      "rewards/margins": 1.1023374795913696,
-      "rewards/rejected": -1.9227107763290405,
-      "step": 1420
-    },
-    {
-      "epoch": 0.8416715715126545,
-      "grad_norm": 48.812190967043605,
-      "learning_rate": 3.7223328814427414e-08,
-      "logits/chosen": -2.7706921100616455,
-      "logits/rejected": -2.6520023345947266,
-      "logps/chosen": -346.84747314453125,
-      "logps/rejected": -356.3780212402344,
-      "loss": 0.4967,
-      "rewards/accuracies": 0.7999999523162842,
-      "rewards/chosen": -0.7521100640296936,
-      "rewards/margins": 0.6091283559799194,
-      "rewards/rejected": -1.3612383604049683,
-      "step": 1430
-    },
-    {
-      "epoch": 0.8475573866980577,
-      "grad_norm": 81.36099469807638,
-      "learning_rate": 3.457171151181154e-08,
-      "logits/chosen": -2.6654701232910156,
-      "logits/rejected": -2.5508956909179688,
-      "logps/chosen": -465.358154296875,
-      "logps/rejected": -375.4325866699219,
-      "loss": 0.5797,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.9726330041885376,
-      "rewards/margins": 0.5235468149185181,
-      "rewards/rejected": -1.4961798191070557,
-      "step": 1440
-    },
-    {
-      "epoch": 0.8534432018834609,
-      "grad_norm": 58.24549104706358,
-      "learning_rate": 3.2011037827024054e-08,
-      "logits/chosen": -2.6841177940368652,
-      "logits/rejected": -2.628507137298584,
-      "logps/chosen": -382.93841552734375,
-      "logps/rejected": -372.9692687988281,
-      "loss": 0.5755,
-      "rewards/accuracies": 0.7333332896232605,
-      "rewards/chosen": -0.9706902503967285,
-      "rewards/margins": 0.5649546384811401,
-      "rewards/rejected": -1.5356448888778687,
-      "step": 1450
-    },
-    {
-      "epoch": 0.859329017068864,
-      "grad_norm": 52.79125544847847,
-      "learning_rate": 2.9542388755186804e-08,
-      "logits/chosen": -2.5700201988220215,
-      "logits/rejected": -2.6171364784240723,
-      "logps/chosen": -345.32354736328125,
-      "logps/rejected": -433.79632568359375,
-      "loss": 0.5057,
-      "rewards/accuracies": 0.7333333492279053,
-      "rewards/chosen": -0.8421486020088196,
-      "rewards/margins": 0.6202816963195801,
-      "rewards/rejected": -1.4624302387237549,
-      "step": 1460
-    },
-    {
-      "epoch": 0.8652148322542672,
-      "grad_norm": 199.09195971504573,
-      "learning_rate": 2.7166806442989743e-08,
-      "logits/chosen": -2.468412399291992,
-      "logits/rejected": -2.5492916107177734,
-      "logps/chosen": -419.80133056640625,
-      "logps/rejected": -409.2911682128906,
-      "loss": 0.5485,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.1756455898284912,
-      "rewards/margins": 0.4044803977012634,
-      "rewards/rejected": -1.5801259279251099,
-      "step": 1470
-    },
-    {
-      "epoch": 0.8711006474396704,
-      "grad_norm": 186.66911649670288,
-      "learning_rate": 2.488529374874557e-08,
-      "logits/chosen": -2.59171724319458,
-      "logits/rejected": -2.5773372650146484,
-      "logps/chosen": -398.6797790527344,
-      "logps/rejected": -408.37554931640625,
-      "loss": 0.4744,
-      "rewards/accuracies": 0.7333332896232605,
-      "rewards/chosen": -0.9879580736160278,
-      "rewards/margins": 0.6637744903564453,
-      "rewards/rejected": -1.6517324447631836,
-      "step": 1480
-    },
-    {
-      "epoch": 0.8769864626250736,
-      "grad_norm": 121.13095753685242,
-      "learning_rate": 2.26988138190308e-08,
-      "logits/chosen": -2.511080265045166,
-      "logits/rejected": -2.476285696029663,
-      "logps/chosen": -388.0962219238281,
-      "logps/rejected": -352.10211181640625,
-      "loss": 0.586,
-      "rewards/accuracies": 0.6666666269302368,
-      "rewards/chosen": -0.7947136163711548,
-      "rewards/margins": 0.43633508682250977,
-      "rewards/rejected": -1.2310487031936646,
-      "step": 1490
-    },
-    {
-      "epoch": 0.8828722778104767,
-      "grad_norm": 60.22203345102361,
-      "learning_rate": 2.0608289682090663e-08,
-      "logits/chosen": -2.488114833831787,
-      "logits/rejected": -2.5074970722198486,
-      "logps/chosen": -347.0746765136719,
-      "logps/rejected": -471.3238220214844,
-      "loss": 0.5907,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.9873822927474976,
-      "rewards/margins": 0.9621238708496094,
-      "rewards/rejected": -1.9495060443878174,
-      "step": 1500
-    },
-    {
-      "epoch": 0.8887580929958799,
-      "grad_norm": 83.04601367710045,
-      "learning_rate": 1.86146038581807e-08,
-      "logits/chosen": -2.6847586631774902,
-      "logits/rejected": -2.6667001247406006,
-      "logps/chosen": -379.8851013183594,
-      "logps/rejected": -402.2454528808594,
-      "loss": 0.5593,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.6927786469459534,
-      "rewards/margins": 0.8660175204277039,
-      "rewards/rejected": -1.5587961673736572,
-      "step": 1510
-    },
-    {
-      "epoch": 0.8946439081812831,
-      "grad_norm": 138.65953465833107,
-      "learning_rate": 1.6718597987007977e-08,
-      "logits/chosen": -2.6951823234558105,
-      "logits/rejected": -2.6642472743988037,
-      "logps/chosen": -458.5162658691406,
-      "logps/rejected": -486.06890869140625,
-      "loss": 0.5285,
-      "rewards/accuracies": 0.7000000476837158,
-      "rewards/chosen": -0.9807785153388977,
-      "rewards/margins": 0.7777236700057983,
-      "rewards/rejected": -1.7585022449493408,
-      "step": 1520
-    },
-    {
-      "epoch": 0.9005297233666862,
-      "grad_norm": 235.95269304208253,
-      "learning_rate": 1.4921072472431073e-08,
-      "logits/chosen": -2.582733392715454,
-      "logits/rejected": -2.54697847366333,
-      "logps/chosen": -380.550048828125,
-      "logps/rejected": -469.8357849121094,
-      "loss": 0.465,
-      "rewards/accuracies": 0.9333333969116211,
-      "rewards/chosen": -0.8054933547973633,
-      "rewards/margins": 1.0806586742401123,
-      "rewards/rejected": -1.8861520290374756,
-      "step": 1530
-    },
-    {
-      "epoch": 0.9064155385520895,
-      "grad_norm": 108.02460840596761,
-      "learning_rate": 1.3222786144567317e-08,
-      "logits/chosen": -2.5321972370147705,
-      "logits/rejected": -2.446223735809326,
-      "logps/chosen": -361.153564453125,
-      "logps/rejected": -472.0574645996094,
-      "loss": 0.5382,
-      "rewards/accuracies": 0.6666666269302368,
-      "rewards/chosen": -1.0409860610961914,
-      "rewards/margins": 0.5477335453033447,
-      "rewards/rejected": -1.5887196063995361,
-      "step": 1540
-    },
-    {
-      "epoch": 0.9123013537374927,
-      "grad_norm": 109.47512454441822,
-      "learning_rate": 1.1624455939450773e-08,
-      "logits/chosen": -2.5635602474212646,
-      "logits/rejected": -2.5231423377990723,
-      "logps/chosen": -390.92144775390625,
-      "logps/rejected": -435.39874267578125,
-      "loss": 0.5468,
-      "rewards/accuracies": 0.76666659116745,
-      "rewards/chosen": -1.122955083847046,
-      "rewards/margins": 0.48533105850219727,
-      "rewards/rejected": -1.6082861423492432,
-      "step": 1550
-    },
-    {
-      "epoch": 0.9181871689228959,
-      "grad_norm": 124.59873217557653,
-      "learning_rate": 1.0126756596375685e-08,
-      "logits/chosen": -2.5917229652404785,
-      "logits/rejected": -2.533984661102295,
-      "logps/chosen": -311.223388671875,
-      "logps/rejected": -396.94775390625,
-      "loss": 0.5317,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -1.0927371978759766,
-      "rewards/margins": 0.5735793113708496,
-      "rewards/rejected": -1.6663163900375366,
-      "step": 1560
-    },
-    {
-      "epoch": 0.924072984108299,
-      "grad_norm": 115.76739079691396,
-      "learning_rate": 8.730320373053545e-09,
-      "logits/chosen": -2.56396484375,
-      "logits/rejected": -2.548893451690674,
-      "logps/chosen": -375.27008056640625,
-      "logps/rejected": -411.82183837890625,
-      "loss": 0.5228,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.8550286293029785,
-      "rewards/margins": 0.5478664636611938,
-      "rewards/rejected": -1.402895212173462,
-      "step": 1570
-    },
-    {
-      "epoch": 0.9299587992937022,
-      "grad_norm": 92.62228298124931,
-      "learning_rate": 7.4357367787039605e-09,
-      "logits/chosen": -2.5038106441497803,
-      "logits/rejected": -2.4670755863189697,
-      "logps/chosen": -403.1292724609375,
-      "logps/rejected": -438.26580810546875,
-      "loss": 0.5464,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.9969646334648132,
-      "rewards/margins": 0.7654547691345215,
-      "rewards/rejected": -1.7624194622039795,
-      "step": 1580
-    },
-    {
-      "epoch": 0.9358446144791054,
-      "grad_norm": 55.929288108030285,
-      "learning_rate": 6.243552325191581e-09,
-      "logits/chosen": -2.4634852409362793,
-      "logits/rejected": -2.364482879638672,
-      "logps/chosen": -391.3017578125,
-      "logps/rejected": -450.46734619140625,
-      "loss": 0.4938,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.8056437373161316,
-      "rewards/margins": 1.2021862268447876,
-      "rewards/rejected": -2.0078299045562744,
-      "step": 1590
-    },
-    {
-      "epoch": 0.9417304296645085,
-      "grad_norm": 61.15914730289007,
-      "learning_rate": 5.154270296314878e-09,
-      "logits/chosen": -2.6435704231262207,
-      "logits/rejected": -2.6495299339294434,
-      "logps/chosen": -374.0923767089844,
-      "logps/rejected": -469.6061096191406,
-      "loss": 0.4876,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.7130493521690369,
-      "rewards/margins": 0.9921830296516418,
-      "rewards/rejected": -1.7052322626113892,
-      "step": 1600
-    },
-    {
-      "epoch": 0.9476162448499117,
-      "grad_norm": 86.60747882561674,
-      "learning_rate": 4.168350535343451e-09,
-      "logits/chosen": -2.5811619758605957,
-      "logits/rejected": -2.5084376335144043,
-      "logps/chosen": -431.181396484375,
-      "logps/rejected": -444.1018981933594,
-      "loss": 0.6032,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.0341752767562866,
-      "rewards/margins": 0.6244133710861206,
-      "rewards/rejected": -1.6585887670516968,
-      "step": 1610
-    },
-    {
-      "epoch": 0.9535020600353149,
-      "grad_norm": 70.5372208774523,
-      "learning_rate": 3.286209250894012e-09,
-      "logits/chosen": -2.616769790649414,
-      "logits/rejected": -2.575296401977539,
-      "logps/chosen": -404.5682373046875,
-      "logps/rejected": -424.28985595703125,
-      "loss": 0.6542,
-      "rewards/accuracies": 0.7666666507720947,
-      "rewards/chosen": -0.8724520802497864,
-      "rewards/margins": 0.5986270308494568,
-      "rewards/rejected": -1.4710791110992432,
-      "step": 1620
-    },
-    {
-      "epoch": 0.9593878752207181,
-      "grad_norm": 88.61728874236027,
-      "learning_rate": 2.5082188412267614e-09,
-      "logits/chosen": -2.624042510986328,
-      "logits/rejected": -2.5592947006225586,
-      "logps/chosen": -363.2521667480469,
-      "logps/rejected": -465.6744079589844,
-      "loss": 0.5553,
-      "rewards/accuracies": 0.7666667103767395,
-      "rewards/chosen": -0.8597677946090698,
-      "rewards/margins": 0.6742967367172241,
-      "rewards/rejected": -1.534064531326294,
-      "step": 1630
-    },
-    {
-      "epoch": 0.9652736904061212,
-      "grad_norm": 108.30056328822583,
-      "learning_rate": 1.8347077370366726e-09,
-      "logits/chosen": -2.449068546295166,
-      "logits/rejected": -2.40153431892395,
-      "logps/chosen": -358.73077392578125,
-      "logps/rejected": -446.489501953125,
-      "loss": 0.5541,
-      "rewards/accuracies": 0.8333333134651184,
-      "rewards/chosen": -0.9515941739082336,
-      "rewards/margins": 0.9860944747924805,
-      "rewards/rejected": -1.9376888275146484,
-      "step": 1640
-    },
-    {
-      "epoch": 0.9711595055915244,
-      "grad_norm": 120.4671832882835,
-      "learning_rate": 1.2659602628053712e-09,
-      "logits/chosen": -2.5365657806396484,
-      "logits/rejected": -2.5613391399383545,
-      "logps/chosen": -341.731689453125,
-      "logps/rejected": -383.644775390625,
-      "loss": 0.5503,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -0.9419547915458679,
-      "rewards/margins": 0.6747300624847412,
-      "rewards/rejected": -1.616684913635254,
-      "step": 1650
-    },
-    {
-      "epoch": 0.9770453207769276,
-      "grad_norm": 91.68610631025756,
-      "learning_rate": 8.022165167730355e-10,
-      "logits/chosen": -2.5892233848571777,
-      "logits/rejected": -2.5990004539489746,
-      "logps/chosen": -303.9766540527344,
-      "logps/rejected": -403.7345275878906,
-      "loss": 0.5349,
-      "rewards/accuracies": 0.6666666269302368,
-      "rewards/chosen": -0.7540067434310913,
-      "rewards/margins": 0.837611198425293,
-      "rewards/rejected": -1.5916178226470947,
-      "step": 1660
-    },
-    {
-      "epoch": 0.9829311359623308,
-      "grad_norm": 48.42706695113977,
-      "learning_rate": 4.4367226958005877e-10,
-      "logits/chosen": -2.6086113452911377,
-      "logits/rejected": -2.5866026878356934,
-      "logps/chosen": -384.8140563964844,
-      "logps/rejected": -449.64776611328125,
-      "loss": 0.5459,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -0.9099410772323608,
-      "rewards/margins": 0.4871155321598053,
-      "rewards/rejected": -1.3970568180084229,
-      "step": 1670
-    },
-    {
-      "epoch": 0.9888169511477339,
-      "grad_norm": 118.15632843194767,
-      "learning_rate": 1.9047888162218584e-10,
-      "logits/chosen": -2.619778633117676,
-      "logits/rejected": -2.5289218425750732,
-      "logps/chosen": -333.962646484375,
-      "logps/rejected": -358.5776672363281,
-      "loss": 0.4757,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.7606975436210632,
-      "rewards/margins": 0.8372971415519714,
-      "rewards/rejected": -1.5979945659637451,
-      "step": 1680
-    },
-    {
-      "epoch": 0.9947027663331371,
-      "grad_norm": 134.31025099698329,
-      "learning_rate": 4.2743239153153474e-11,
-      "logits/chosen": -2.48134708404541,
-      "logits/rejected": -2.3866162300109863,
-      "logps/chosen": -300.18408203125,
-      "logps/rejected": -355.9591369628906,
-      "loss": 0.5545,
-      "rewards/accuracies": 0.6333333253860474,
-      "rewards/chosen": -1.1656824350357056,
-      "rewards/margins": 0.4039548337459564,
-      "rewards/rejected": -1.5696371793746948,
-      "step": 1690
-    },
-    {
-      "epoch": 1.0,
-      "step": 1699,
+      "epoch": 0.9984301412872841,
+      "step": 159,
       "total_flos": 0.0,
-      "train_loss": 0.03212777637888081,
-      "train_runtime": 304.206,
-      "train_samples_per_second": 50.239,
-      "train_steps_per_second": 5.585
+      "train_loss": 0.6094696866641255,
+      "train_runtime": 4741.6763,
+      "train_samples_per_second": 3.223,
+      "train_steps_per_second": 0.034
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1699,
+  "max_steps": 159,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
@@ -2586,7 +276,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 3,
+  "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null
 }