Upload folder using huggingface_hub

all_results.json

@@ -1,24 +1,24 @@
 {
     "epoch": 0.9959925193694897,
-    "eval_logits/chosen": -2.511686325073242,
-    "eval_logits/rejected": -2.44095516204834,
-    "eval_logps/chosen": -0.6983144879341125,
-    "eval_logps/ref_chosen": -0.7951558232307434,
-    "eval_logps/ref_rejected": -0.8243172764778137,
-    "eval_logps/rejected": -0.8109735250473022,
-    "eval_loss": 0.4769956171512604,
-    "eval_rewards/accuracies": 0.6814516186714172,
-    "eval_rewards/chosen": 0.24210312962532043,
-    "eval_rewards/margins": 0.20874378085136414,
-    "eval_rewards/rejected": 0.033359345048666,
-    "eval_runtime": 145.1612,
+    "eval_logits/chosen": -2.370966672897339,
+    "eval_logits/rejected": -2.2963974475860596,
+    "eval_logps/chosen": -223.0048065185547,
+    "eval_logps/ref_chosen": -237.70138549804688,
+    "eval_logps/ref_rejected": -244.81951904296875,
+    "eval_logps/rejected": -232.80235290527344,
+    "eval_loss": 0.4936164915561676,
+    "eval_rewards/accuracies": 0.6572580933570862,
+    "eval_rewards/chosen": 0.14696598052978516,
+    "eval_rewards/margins": 0.026794558390975,
+    "eval_rewards/rejected": 0.12017140537500381,
+    "eval_runtime": 141.5135,
     "eval_samples": 1961,
-    "eval_samples_per_second": 13.509,
-    "eval_steps_per_second": 0.427,
+    "eval_samples_per_second": 13.857,
+    "eval_steps_per_second": 0.438,
     "total_flos": 0.0,
-    "train_loss": 0.4860667634931245,
-    "train_runtime": 16529.3176,
+    "train_loss": 0.49642937480124283,
+    "train_runtime": 16410.2083,
     "train_samples": 59876,
-    "train_samples_per_second": 3.622,
+    "train_samples_per_second": 3.649,
     "train_steps_per_second": 0.014
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 0.9959925193694897,
-    "eval_logits/chosen": -2.511686325073242,
-    "eval_logits/rejected": -2.44095516204834,
-    "eval_logps/chosen": -0.6983144879341125,
-    "eval_logps/ref_chosen": -0.7951558232307434,
-    "eval_logps/ref_rejected": -0.8243172764778137,
-    "eval_logps/rejected": -0.8109735250473022,
-    "eval_loss": 0.4769956171512604,
-    "eval_rewards/accuracies": 0.6814516186714172,
-    "eval_rewards/chosen": 0.24210312962532043,
-    "eval_rewards/margins": 0.20874378085136414,
-    "eval_rewards/rejected": 0.033359345048666,
-    "eval_runtime": 145.1612,
+    "eval_logits/chosen": -2.370966672897339,
+    "eval_logits/rejected": -2.2963974475860596,
+    "eval_logps/chosen": -223.0048065185547,
+    "eval_logps/ref_chosen": -237.70138549804688,
+    "eval_logps/ref_rejected": -244.81951904296875,
+    "eval_logps/rejected": -232.80235290527344,
+    "eval_loss": 0.4936164915561676,
+    "eval_rewards/accuracies": 0.6572580933570862,
+    "eval_rewards/chosen": 0.14696598052978516,
+    "eval_rewards/margins": 0.026794558390975,
+    "eval_rewards/rejected": 0.12017140537500381,
+    "eval_runtime": 141.5135,
     "eval_samples": 1961,
-    "eval_samples_per_second": 13.509,
-    "eval_steps_per_second": 0.427
+    "eval_samples_per_second": 13.857,
+    "eval_steps_per_second": 0.438
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0fcc07c77caf38cf16eca94466b09a450f59047949056da639891d7db810679
+oid sha256:275ce3f1a86067d54cdf9833ffb6bc6744cc7f069a565035201c3b5532a681e0
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:716ddb06194501c683fce829c9c12c4360bffbe234af9790616bb8dea15c9a7c
+oid sha256:357ef4d1e8076cf6ed25ef0832b4f778ccaac92509a671be2f0bd459bec774f9
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac1c3d736e0f1ea37f82f87d7e8a5ce546e8731b7d6e863adb987c73e40b1482
+oid sha256:3f3bf313ea09793309e6446a9f39cb995390296792fb6bafc05f63c2e6b81cac
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1848cd805e8117e7c2d8bce0d5fa2d9a2ea04fcef2a0930d658566669435ea46
+oid sha256:46028d9aa6fb09491ef46a7ca1c69c42eb69eb801c9c83cc2063a1dc0ccb22c2
 size 1168138808

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9959925193694897,
     "total_flos": 0.0,
-    "train_loss": 0.4860667634931245,
-    "train_runtime": 16529.3176,
+    "train_loss": 0.49642937480124283,
+    "train_runtime": 16410.2083,
     "train_samples": 59876,
-    "train_samples_per_second": 3.622,
+    "train_samples_per_second": 3.649,
     "train_steps_per_second": 0.014
 }

trainer_state.json

@@ -10,793 +10,793 @@
   "log_history": [
     {
       "epoch": 0.02137323002938819,
-      "grad_norm": 0.7561385035514832,
-      "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -1.7804944515228271,
-      "logits/rejected": -1.6545133590698242,
-      "logps/chosen": -0.7810468673706055,
-      "logps/ref_chosen": -0.7813535928726196,
-      "logps/ref_rejected": -0.8060104250907898,
-      "logps/rejected": -0.805632472038269,
+      "grad_norm": 0.4608515202999115,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -1.7747037410736084,
+      "logits/rejected": -1.6486629247665405,
+      "logps/chosen": -247.47836303710938,
+      "logps/ref_chosen": -247.4757537841797,
+      "logps/ref_rejected": -250.2177734375,
+      "logps/rejected": -250.17874145507812,
       "loss": 0.5,
-      "rewards/accuracies": 0.33125001192092896,
-      "rewards/chosen": 0.0007667625322937965,
-      "rewards/margins": -0.00017807073891162872,
-      "rewards/rejected": 0.0009448332712054253,
+      "rewards/accuracies": 0.29374998807907104,
+      "rewards/chosen": -2.605724148452282e-05,
+      "rewards/margins": -0.00041639525443315506,
+      "rewards/rejected": 0.0003903379547409713,
       "step": 5
     },
     {
       "epoch": 0.04274646005877638,
-      "grad_norm": 0.5518713593482971,
-      "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -1.7383073568344116,
-      "logits/rejected": -1.7030198574066162,
-      "logps/chosen": -0.8675562143325806,
-      "logps/ref_chosen": -0.8662088513374329,
-      "logps/ref_rejected": -0.9053529500961304,
-      "logps/rejected": -0.907278835773468,
+      "grad_norm": 0.426495224237442,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -1.7335236072540283,
+      "logits/rejected": -1.6989978551864624,
+      "logps/chosen": -222.6909637451172,
+      "logps/ref_chosen": -222.6491241455078,
+      "logps/ref_rejected": -223.95663452148438,
+      "logps/rejected": -223.9930877685547,
       "loss": 0.5,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.003368514124304056,
-      "rewards/margins": 0.0014460685197263956,
-      "rewards/rejected": -0.004814582876861095,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.00041838129982352257,
+      "rewards/margins": -5.400222653406672e-05,
+      "rewards/rejected": -0.00036437893868424,
       "step": 10
     },
     {
       "epoch": 0.06411969008816458,
-      "grad_norm": 0.49651747941970825,
-      "learning_rate": 6.249999999999999e-07,
-      "logits/chosen": -1.928145170211792,
-      "logits/rejected": -1.8129940032958984,
-      "logps/chosen": -0.855307400226593,
-      "logps/ref_chosen": -0.8494969606399536,
-      "logps/ref_rejected": -0.8662179708480835,
-      "logps/rejected": -0.8723622560501099,
-      "loss": 0.4998,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.014526228420436382,
-      "rewards/margins": 0.0008344631642103195,
-      "rewards/rejected": -0.015360690653324127,
+      "grad_norm": 0.4453659653663635,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -1.9023773670196533,
+      "logits/rejected": -1.789849042892456,
+      "logps/chosen": -218.5724334716797,
+      "logps/ref_chosen": -218.7084503173828,
+      "logps/ref_rejected": -224.755615234375,
+      "logps/rejected": -224.6824493408203,
+      "loss": 0.5,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0013600520323961973,
+      "rewards/margins": 0.0006284656701609492,
+      "rewards/rejected": 0.000731586420442909,
       "step": 15
     },
     {
       "epoch": 0.08549292011755276,
-      "grad_norm": 0.27864935994148254,
-      "learning_rate": 8.333333333333333e-07,
-      "logits/chosen": -1.7597744464874268,
-      "logits/rejected": -1.6725934743881226,
-      "logps/chosen": -0.9120359420776367,
-      "logps/ref_chosen": -0.8935796618461609,
-      "logps/ref_rejected": -0.8952409029006958,
-      "logps/rejected": -0.9148454666137695,
-      "loss": 0.4993,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.0461406409740448,
-      "rewards/margins": 0.0028707808814942837,
-      "rewards/rejected": -0.04901142045855522,
+      "grad_norm": 0.5101017951965332,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -1.7127611637115479,
+      "logits/rejected": -1.6293315887451172,
+      "logps/chosen": -226.1074676513672,
+      "logps/ref_chosen": -226.7457275390625,
+      "logps/ref_rejected": -235.77908325195312,
+      "logps/rejected": -235.2657928466797,
+      "loss": 0.4999,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.006382433231920004,
+      "rewards/margins": 0.0012494683032855392,
+      "rewards/rejected": 0.005132964812219143,
       "step": 20
     },
     {
       "epoch": 0.10686615014694095,
-      "grad_norm": 0.4763440787792206,
-      "learning_rate": 9.999435142363483e-07,
-      "logits/chosen": -1.624091386795044,
-      "logits/rejected": -1.5722483396530151,
-      "logps/chosen": -0.9541429281234741,
-      "logps/ref_chosen": -0.8983734846115112,
-      "logps/ref_rejected": -0.9594888687133789,
-      "logps/rejected": -1.035103440284729,
-      "loss": 0.4976,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.1394234150648117,
-      "rewards/margins": 0.04961305111646652,
-      "rewards/rejected": -0.18903647363185883,
+      "grad_norm": 0.4738335609436035,
+      "learning_rate": 4.999717571181741e-07,
+      "logits/chosen": -1.6099249124526978,
+      "logits/rejected": -1.5539109706878662,
+      "logps/chosen": -229.36843872070312,
+      "logps/ref_chosen": -230.34494018554688,
+      "logps/ref_rejected": -231.64236450195312,
+      "logps/rejected": -230.74813842773438,
+      "loss": 0.4999,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.009765096008777618,
+      "rewards/margins": 0.000822968955617398,
+      "rewards/rejected": 0.008942126296460629,
       "step": 25
     },
     {
       "epoch": 0.12823938017632916,
-      "grad_norm": 0.3390841782093048,
-      "learning_rate": 9.97967852255038e-07,
-      "logits/chosen": -1.6577975749969482,
-      "logits/rejected": -1.5775320529937744,
-      "logps/chosen": -0.8317171931266785,
-      "logps/ref_chosen": -0.7469085454940796,
-      "logps/ref_rejected": -0.79144287109375,
-      "logps/rejected": -0.8856765031814575,
-      "loss": 0.4975,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.2120215892791748,
-      "rewards/margins": 0.023562394082546234,
-      "rewards/rejected": -0.23558397591114044,
+      "grad_norm": 0.4367460608482361,
+      "learning_rate": 4.98983926127519e-07,
+      "logits/chosen": -1.6448577642440796,
+      "logits/rejected": -1.560329794883728,
+      "logps/chosen": -239.9384002685547,
+      "logps/ref_chosen": -241.2040557861328,
+      "logps/ref_rejected": -253.18862915039062,
+      "logps/rejected": -251.95547485351562,
+      "loss": 0.4998,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.012656150385737419,
+      "rewards/margins": 0.00032438611378893256,
+      "rewards/rejected": 0.012331764213740826,
       "step": 30
     },
     {
       "epoch": 0.14961261020571734,
-      "grad_norm": 0.3006940484046936,
-      "learning_rate": 9.931806517013612e-07,
-      "logits/chosen": -1.6243362426757812,
-      "logits/rejected": -1.6471837759017944,
-      "logps/chosen": -0.9220904111862183,
-      "logps/ref_chosen": -0.7822158336639404,
-      "logps/ref_rejected": -0.8102364540100098,
-      "logps/rejected": -0.9670912027359009,
-      "loss": 0.4963,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.34968677163124084,
-      "rewards/margins": 0.04244992882013321,
-      "rewards/rejected": -0.39213672280311584,
+      "grad_norm": 0.5036317706108093,
+      "learning_rate": 4.965903258506806e-07,
+      "logits/chosen": -1.65009343624115,
+      "logits/rejected": -1.6685165166854858,
+      "logps/chosen": -240.6787109375,
+      "logps/ref_chosen": -242.33291625976562,
+      "logps/ref_rejected": -237.6911163330078,
+      "logps/rejected": -236.1189422607422,
+      "loss": 0.4997,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.016541726887226105,
+      "rewards/margins": 0.0008201012387871742,
+      "rewards/rejected": 0.015721624717116356,
       "step": 35
     },
     {
       "epoch": 0.17098584023510552,
-      "grad_norm": 0.27084407210350037,
-      "learning_rate": 9.856089412257604e-07,
-      "logits/chosen": -1.6578766107559204,
-      "logits/rejected": -1.6355148553848267,
-      "logps/chosen": -1.05038583278656,
-      "logps/ref_chosen": -0.8560595512390137,
-      "logps/ref_rejected": -0.914546012878418,
-      "logps/rejected": -1.1552728414535522,
-      "loss": 0.4937,
+      "grad_norm": 0.5212914347648621,
+      "learning_rate": 4.928044706128802e-07,
+      "logits/chosen": -1.6572792530059814,
+      "logits/rejected": -1.6342990398406982,
+      "logps/chosen": -224.078857421875,
+      "logps/ref_chosen": -226.43637084960938,
+      "logps/ref_rejected": -224.00546264648438,
+      "logps/rejected": -221.7003173828125,
+      "loss": 0.4996,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.48581594228744507,
-      "rewards/margins": 0.11600111424922943,
-      "rewards/rejected": -0.6018170118331909,
+      "rewards/chosen": 0.02357516996562481,
+      "rewards/margins": 0.0005238516023382545,
+      "rewards/rejected": 0.023051317781209946,
       "step": 40
     },
     {
       "epoch": 0.19235907026449373,
-      "grad_norm": 0.49249762296676636,
-      "learning_rate": 9.752954708892377e-07,
-      "logits/chosen": -1.5577068328857422,
-      "logits/rejected": -1.465714931488037,
-      "logps/chosen": -1.0501822233200073,
-      "logps/ref_chosen": -0.8724653124809265,
-      "logps/ref_rejected": -0.8607926368713379,
-      "logps/rejected": -1.0398765802383423,
-      "loss": 0.4975,
+      "grad_norm": 0.5110143423080444,
+      "learning_rate": 4.876477354446189e-07,
+      "logits/chosen": -1.4905364513397217,
+      "logits/rejected": -1.3957011699676514,
+      "logps/chosen": -216.25308227539062,
+      "logps/ref_chosen": -219.16494750976562,
+      "logps/ref_rejected": -227.38040161132812,
+      "logps/rejected": -224.87564086914062,
+      "loss": 0.4994,
       "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.4442923665046692,
-      "rewards/margins": 0.003417615545913577,
-      "rewards/rejected": -0.4477098882198334,
+      "rewards/chosen": 0.02911846712231636,
+      "rewards/margins": 0.004071122966706753,
+      "rewards/rejected": 0.025047341361641884,
       "step": 45
     },
     {
       "epoch": 0.2137323002938819,
-      "grad_norm": 0.32699063420295715,
-      "learning_rate": 9.62298470795473e-07,
-      "logits/chosen": -1.7691097259521484,
-      "logits/rejected": -1.7416681051254272,
-      "logps/chosen": -0.9927698969841003,
-      "logps/ref_chosen": -0.8696678280830383,
-      "logps/ref_rejected": -0.8965504765510559,
-      "logps/rejected": -1.030956506729126,
-      "loss": 0.4967,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.3077549934387207,
-      "rewards/margins": 0.028260568156838417,
-      "rewards/rejected": -0.33601561188697815,
+      "grad_norm": 0.48523762822151184,
+      "learning_rate": 4.811492353977365e-07,
+      "logits/chosen": -1.7010364532470703,
+      "logits/rejected": -1.6736198663711548,
+      "logps/chosen": -218.8837127685547,
+      "logps/ref_chosen": -221.23171997070312,
+      "logps/ref_rejected": -223.6177215576172,
+      "logps/rejected": -221.6636199951172,
+      "loss": 0.4993,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.023480093106627464,
+      "rewards/margins": 0.0039388458244502544,
+      "rewards/rejected": 0.019541248679161072,
       "step": 50
     },
     {
       "epoch": 0.2351055303232701,
-      "grad_norm": 0.31701868772506714,
-      "learning_rate": 9.466913223222465e-07,
-      "logits/chosen": -1.5519920587539673,
-      "logits/rejected": -1.4699208736419678,
-      "logps/chosen": -0.8616452217102051,
-      "logps/ref_chosen": -0.7731812596321106,
-      "logps/ref_rejected": -0.7838868498802185,
-      "logps/rejected": -0.8863974809646606,
-      "loss": 0.4956,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.22115974128246307,
-      "rewards/margins": 0.03511647880077362,
-      "rewards/rejected": -0.2562762200832367,
+      "grad_norm": 0.4816797971725464,
+      "learning_rate": 4.7334566116112327e-07,
+      "logits/chosen": -1.62349534034729,
+      "logits/rejected": -1.5281016826629639,
+      "logps/chosen": -237.206787109375,
+      "logps/ref_chosen": -239.38412475585938,
+      "logps/ref_rejected": -245.71304321289062,
+      "logps/rejected": -244.2113800048828,
+      "loss": 0.4989,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.021773329004645348,
+      "rewards/margins": 0.006756873335689306,
+      "rewards/rejected": 0.015016456134617329,
       "step": 55
     },
     {
       "epoch": 0.2564787603526583,
-      "grad_norm": 0.40317973494529724,
-      "learning_rate": 9.285621438083997e-07,
-      "logits/chosen": -1.601485252380371,
-      "logits/rejected": -1.5545583963394165,
-      "logps/chosen": -0.8779473304748535,
-      "logps/ref_chosen": -0.7888692617416382,
-      "logps/ref_rejected": -0.8163660168647766,
-      "logps/rejected": -0.9202233552932739,
-      "loss": 0.4936,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.22269515693187714,
-      "rewards/margins": 0.03694819286465645,
-      "rewards/rejected": -0.2596433460712433,
+      "grad_norm": 0.5273976922035217,
+      "learning_rate": 4.6428107190419983e-07,
+      "logits/chosen": -1.6468950510025024,
+      "logits/rejected": -1.599461317062378,
+      "logps/chosen": -228.3268585205078,
+      "logps/ref_chosen": -231.1789093017578,
+      "logps/ref_rejected": -231.9095001220703,
+      "logps/rejected": -229.9440460205078,
+      "loss": 0.4988,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.028520625084638596,
+      "rewards/margins": 0.008865959011018276,
+      "rewards/rejected": 0.019654670730233192,
       "step": 60
     },
     {
       "epoch": 0.2778519903820465,
-      "grad_norm": 0.5152870416641235,
-      "learning_rate": 9.080132930355566e-07,
-      "logits/chosen": -1.6490017175674438,
-      "logits/rejected": -1.6716206073760986,
-      "logps/chosen": -0.9653270840644836,
-      "logps/ref_chosen": -0.8533055186271667,
-      "logps/ref_rejected": -0.9036076664924622,
-      "logps/rejected": -1.0383471250534058,
-      "loss": 0.4942,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.28005388379096985,
-      "rewards/margins": 0.05679459124803543,
-      "rewards/rejected": -0.33684849739074707,
+      "grad_norm": 0.47698166966438293,
+      "learning_rate": 4.540066465177783e-07,
+      "logits/chosen": -1.7030376195907593,
+      "logits/rejected": -1.7270011901855469,
+      "logps/chosen": -218.37466430664062,
+      "logps/ref_chosen": -222.1732635498047,
+      "logps/ref_rejected": -221.90371704101562,
+      "logps/rejected": -219.0262451171875,
+      "loss": 0.4985,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.03798612207174301,
+      "rewards/margins": 0.009211419150233269,
+      "rewards/rejected": 0.028774702921509743,
       "step": 65
     },
     {
       "epoch": 0.2992252204114347,
-      "grad_norm": 0.4174951910972595,
-      "learning_rate": 8.851607893136064e-07,
-      "logits/chosen": -1.728899598121643,
-      "logits/rejected": -1.6759620904922485,
-      "logps/chosen": -0.9261114001274109,
-      "logps/ref_chosen": -0.8132463693618774,
-      "logps/ref_rejected": -0.8208681344985962,
-      "logps/rejected": -0.9566439390182495,
-      "loss": 0.4926,
+      "grad_norm": 0.4908115863800049,
+      "learning_rate": 4.425803946568032e-07,
+      "logits/chosen": -1.701042890548706,
+      "logits/rejected": -1.642853021621704,
+      "logps/chosen": -237.1160430908203,
+      "logps/ref_chosen": -241.13235473632812,
+      "logps/ref_rejected": -247.3893585205078,
+      "logps/rejected": -243.56692504882812,
+      "loss": 0.4985,
       "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.28216248750686646,
-      "rewards/margins": 0.057276882231235504,
-      "rewards/rejected": -0.33943939208984375,
+      "rewards/chosen": 0.040162790566682816,
+      "rewards/margins": 0.0019384495681151748,
+      "rewards/rejected": 0.038224343210458755,
       "step": 70
     },
     {
       "epoch": 0.32059845044082286,
-      "grad_norm": 0.36295104026794434,
-      "learning_rate": 8.601336584328658e-07,
-      "logits/chosen": -1.7176015377044678,
-      "logits/rejected": -1.7168267965316772,
-      "logps/chosen": -0.9694639444351196,
-      "logps/ref_chosen": -0.8283951878547668,
-      "logps/ref_rejected": -0.8723212480545044,
-      "logps/rejected": -1.0357882976531982,
-      "loss": 0.4939,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.3526715338230133,
-      "rewards/margins": 0.05599608272314072,
-      "rewards/rejected": -0.4086676239967346,
+      "grad_norm": 0.48811107873916626,
+      "learning_rate": 4.300668292164329e-07,
+      "logits/chosen": -1.6175544261932373,
+      "logits/rejected": -1.6155774593353271,
+      "logps/chosen": -223.8777618408203,
+      "logps/ref_chosen": -228.91860961914062,
+      "logps/ref_rejected": -227.78170776367188,
+      "logps/rejected": -223.22732543945312,
+      "loss": 0.4981,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.05040856450796127,
+      "rewards/margins": 0.004864625167101622,
+      "rewards/rejected": 0.04554395005106926,
       "step": 75
     },
     {
       "epoch": 0.34197168047021104,
-      "grad_norm": 0.6786319017410278,
-      "learning_rate": 8.330732041813366e-07,
-      "logits/chosen": -1.8271814584732056,
-      "logits/rejected": -1.7772512435913086,
-      "logps/chosen": -0.8929777145385742,
-      "logps/ref_chosen": -0.8355891108512878,
-      "logps/ref_rejected": -0.9002590179443359,
-      "logps/rejected": -0.9975314140319824,
-      "loss": 0.4901,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.14347167313098907,
-      "rewards/margins": 0.09970954060554504,
-      "rewards/rejected": -0.24318119883537292,
+      "grad_norm": 0.5498376488685608,
+      "learning_rate": 4.165366020906683e-07,
+      "logits/chosen": -1.721421480178833,
+      "logits/rejected": -1.6703542470932007,
+      "logps/chosen": -220.573486328125,
+      "logps/ref_chosen": -226.90060424804688,
+      "logps/ref_rejected": -232.0827178955078,
+      "logps/rejected": -227.0341339111328,
+      "loss": 0.4975,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.06327112019062042,
+      "rewards/margins": 0.012785114347934723,
+      "rewards/rejected": 0.0504860058426857,
       "step": 80
     },
     {
       "epoch": 0.36334491049959927,
-      "grad_norm": 0.8407174348831177,
-      "learning_rate": 8.041322105400921e-07,
-      "logits/chosen": -1.706368088722229,
-      "logits/rejected": -1.650854468345642,
-      "logps/chosen": -0.8318307995796204,
-      "logps/ref_chosen": -0.8256785273551941,
-      "logps/ref_rejected": -0.8488883972167969,
-      "logps/rejected": -0.9011926651000977,
-      "loss": 0.4878,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.01538090594112873,
-      "rewards/margins": 0.1153799295425415,
-      "rewards/rejected": -0.13076083362102509,
+      "grad_norm": 0.5343174338340759,
+      "learning_rate": 4.0206610527004607e-07,
+      "logits/chosen": -1.630051612854004,
+      "logits/rejected": -1.571542739868164,
+      "logps/chosen": -231.68496704101562,
+      "logps/ref_chosen": -237.4697723388672,
+      "logps/ref_rejected": -240.751953125,
+      "logps/rejected": -236.31600952148438,
+      "loss": 0.4978,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.05784807354211807,
+      "rewards/margins": 0.013488592579960823,
+      "rewards/rejected": 0.0443594828248024,
       "step": 85
     },
     {
       "epoch": 0.38471814052898745,
-      "grad_norm": 1.0095113515853882,
-      "learning_rate": 7.734740790612136e-07,
-      "logits/chosen": -1.8660595417022705,
-      "logits/rejected": -1.8641777038574219,
-      "logps/chosen": -0.8596251606941223,
-      "logps/ref_chosen": -0.9228288531303406,
-      "logps/ref_rejected": -0.9406684637069702,
-      "logps/rejected": -0.9123810529708862,
-      "loss": 0.4896,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.15800921618938446,
-      "rewards/margins": 0.08729076385498047,
-      "rewards/rejected": 0.07071846723556519,
+      "grad_norm": 0.5112692713737488,
+      "learning_rate": 3.867370395306068e-07,
+      "logits/chosen": -1.7595088481903076,
+      "logits/rejected": -1.7580636739730835,
+      "logps/chosen": -211.63906860351562,
+      "logps/ref_chosen": -217.63436889648438,
+      "logps/ref_rejected": -222.6137237548828,
+      "logps/rejected": -217.2650909423828,
+      "loss": 0.4977,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.05995314195752144,
+      "rewards/margins": 0.00646712351590395,
+      "rewards/rejected": 0.053486019372940063,
       "step": 90
     },
     {
       "epoch": 0.40609137055837563,
-      "grad_norm": 0.8767968416213989,
-      "learning_rate": 7.412719062986631e-07,
-      "logits/chosen": -1.9249579906463623,
-      "logits/rejected": -1.8531602621078491,
-      "logps/chosen": -0.8149619102478027,
-      "logps/ref_chosen": -0.9041957855224609,
-      "logps/ref_rejected": -0.914394736289978,
-      "logps/rejected": -0.8845084309577942,
-      "loss": 0.4859,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.22308464348316193,
-      "rewards/margins": 0.14836890995502472,
-      "rewards/rejected": 0.0747157484292984,
+      "grad_norm": 0.4654058516025543,
+      "learning_rate": 3.7063595314933156e-07,
+      "logits/chosen": -1.8619199991226196,
+      "logits/rejected": -1.786892294883728,
+      "logps/chosen": -208.5725555419922,
+      "logps/ref_chosen": -213.7164306640625,
+      "logps/ref_rejected": -228.556396484375,
+      "logps/rejected": -224.4815216064453,
+      "loss": 0.498,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.051438819617033005,
+      "rewards/margins": 0.010690188966691494,
+      "rewards/rejected": 0.04074862599372864,
       "step": 95
     },
     {
       "epoch": 0.4274646005877638,
-      "grad_norm": 0.6044840812683105,
-      "learning_rate": 7.077075065009433e-07,
-      "logits/chosen": -1.731792688369751,
-      "logits/rejected": -1.7363353967666626,
-      "logps/chosen": -0.7217603921890259,
-      "logps/ref_chosen": -0.8257284164428711,
-      "logps/ref_rejected": -0.8479409217834473,
-      "logps/rejected": -0.784611701965332,
-      "loss": 0.4852,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.2599199414253235,
-      "rewards/margins": 0.10159693658351898,
-      "rewards/rejected": 0.1583230048418045,
+      "grad_norm": 0.5265087485313416,
+      "learning_rate": 3.5385375325047163e-07,
+      "logits/chosen": -1.6727230548858643,
+      "logits/rejected": -1.677062749862671,
+      "logps/chosen": -239.5093994140625,
+      "logps/ref_chosen": -245.71194458007812,
+      "logps/ref_rejected": -240.1134490966797,
+      "logps/rejected": -235.6671142578125,
+      "loss": 0.4967,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.06202547624707222,
+      "rewards/margins": 0.01756184920668602,
+      "rewards/rejected": 0.0444636233150959,
       "step": 100
     },
     {
       "epoch": 0.448837830617152,
-      "grad_norm": 0.7751099467277527,
-      "learning_rate": 6.72970385083438e-07,
-      "logits/chosen": -1.9043171405792236,
-      "logits/rejected": -1.789009690284729,
-      "logps/chosen": -0.7011796236038208,
-      "logps/ref_chosen": -0.8166704177856445,
-      "logps/ref_rejected": -0.8361040949821472,
-      "logps/rejected": -0.7531259655952454,
-      "loss": 0.4905,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.2887269854545593,
-      "rewards/margins": 0.0812816247344017,
-      "rewards/rejected": 0.20744535326957703,
+      "grad_norm": 0.53775554895401,
+      "learning_rate": 3.36485192541719e-07,
+      "logits/chosen": -1.8463099002838135,
+      "logits/rejected": -1.7264705896377563,
+      "logps/chosen": -224.50320434570312,
+      "logps/ref_chosen": -232.00527954101562,
+      "logps/ref_rejected": -232.0154266357422,
+      "logps/rejected": -225.75454711914062,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0750209242105484,
+      "rewards/margins": 0.012411920353770256,
+      "rewards/rejected": 0.062609001994133,
       "step": 105
     },
     {
       "epoch": 0.4702110606465402,
-      "grad_norm": 0.7444783449172974,
-      "learning_rate": 6.372566686762426e-07,
-      "logits/chosen": -1.8429124355316162,
-      "logits/rejected": -1.760053277015686,
-      "logps/chosen": -0.7318671941757202,
-      "logps/ref_chosen": -0.8331576585769653,
-      "logps/ref_rejected": -0.8635438084602356,
-      "logps/rejected": -0.7953906059265137,
-      "loss": 0.4817,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.253226101398468,
-      "rewards/margins": 0.08284299075603485,
-      "rewards/rejected": 0.17038312554359436,
+      "grad_norm": 0.5438077449798584,
+      "learning_rate": 3.186283343381213e-07,
+      "logits/chosen": -1.7997539043426514,
+      "logits/rejected": -1.7138378620147705,
+      "logps/chosen": -220.4825897216797,
+      "logps/ref_chosen": -229.9724578857422,
+      "logps/ref_rejected": -238.1800079345703,
+      "logps/rejected": -230.29736328125,
+      "loss": 0.4966,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.09489865601062775,
+      "rewards/margins": 0.016072329133749008,
+      "rewards/rejected": 0.07882632315158844,
       "step": 110
     },
     {
       "epoch": 0.4915842906759284,
-      "grad_norm": 0.8039557337760925,
-      "learning_rate": 6.00767997788451e-07,
-      "logits/chosen": -1.9033887386322021,
-      "logits/rejected": -1.770939588546753,
-      "logps/chosen": -0.7566145062446594,
-      "logps/ref_chosen": -0.8713752627372742,
-      "logps/ref_rejected": -0.8939735293388367,
-      "logps/rejected": -0.8082603216171265,
-      "loss": 0.4844,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.28690171241760254,
-      "rewards/margins": 0.07261888682842255,
-      "rewards/rejected": 0.2142828404903412,
+      "grad_norm": 0.5453912019729614,
+      "learning_rate": 3.003839988942255e-07,
+      "logits/chosen": -1.8438644409179688,
+      "logits/rejected": -1.7028881311416626,
+      "logps/chosen": -203.79205322265625,
+      "logps/ref_chosen": -214.1478729248047,
+      "logps/ref_rejected": -226.24618530273438,
+      "logps/rejected": -217.4800567626953,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.1035580188035965,
+      "rewards/margins": 0.015896398574113846,
+      "rewards/rejected": 0.08766160905361176,
       "step": 115
     },
     {
       "epoch": 0.5129575207053166,
-      "grad_norm": 1.1172226667404175,
-      "learning_rate": 5.637103883409525e-07,
-      "logits/chosen": -1.9406812191009521,
-      "logits/rejected": -1.867462158203125,
-      "logps/chosen": -0.8249004483222961,
-      "logps/ref_chosen": -0.873686671257019,
-      "logps/ref_rejected": -0.9026174545288086,
-      "logps/rejected": -0.8801124691963196,
-      "loss": 0.4836,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.1219654530286789,
-      "rewards/margins": 0.06570279598236084,
-      "rewards/rejected": 0.05626266077160835,
+      "grad_norm": 0.5030398964881897,
+      "learning_rate": 2.8185519417047623e-07,
+      "logits/chosen": -1.8514922857284546,
+      "logits/rejected": -1.7740070819854736,
+      "logps/chosen": -214.818359375,
+      "logps/ref_chosen": -227.9495086669922,
+      "logps/ref_rejected": -230.5752410888672,
+      "logps/rejected": -218.9449005126953,
+      "loss": 0.496,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.13131138682365417,
+      "rewards/margins": 0.015008069574832916,
+      "rewards/rejected": 0.11630330979824066,
       "step": 120
     },
     {
       "epoch": 0.5343307507347048,
-      "grad_norm": 0.8093484044075012,
-      "learning_rate": 5.262930684955438e-07,
-      "logits/chosen": -2.0165348052978516,
-      "logits/rejected": -1.9574447870254517,
-      "logps/chosen": -0.8015368580818176,
-      "logps/ref_chosen": -0.815376877784729,
-      "logps/ref_rejected": -0.8817696571350098,
-      "logps/rejected": -0.9249275326728821,
-      "loss": 0.4825,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.034600116312503815,
-      "rewards/margins": 0.14249476790428162,
-      "rewards/rejected": -0.1078946590423584,
+      "grad_norm": 0.5339066982269287,
+      "learning_rate": 2.631465342477719e-07,
+      "logits/chosen": -1.9007892608642578,
+      "logits/rejected": -1.8334102630615234,
+      "logps/chosen": -218.14743041992188,
+      "logps/ref_chosen": -232.6212158203125,
+      "logps/ref_rejected": -234.5932159423828,
+      "logps/rejected": -222.1468505859375,
+      "loss": 0.4958,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.1447378695011139,
+      "rewards/margins": 0.020274382084608078,
+      "rewards/rejected": 0.12446349859237671,
       "step": 125
     },
     {
       "epoch": 0.555703980764093,
-      "grad_norm": 0.8614518642425537,
-      "learning_rate": 4.88727297347654e-07,
-      "logits/chosen": -1.951319694519043,
-      "logits/rejected": -1.933098554611206,
-      "logps/chosen": -0.7757576107978821,
-      "logps/ref_chosen": -0.7751168608665466,
-      "logps/ref_rejected": -0.8734749555587769,
-      "logps/rejected": -0.9476582407951355,
-      "loss": 0.4794,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.0016015321016311646,
-      "rewards/margins": 0.18385668098926544,
-      "rewards/rejected": -0.1854582130908966,
+      "grad_norm": 0.5313855409622192,
+      "learning_rate": 2.44363648673827e-07,
+      "logits/chosen": -1.7636210918426514,
+      "logits/rejected": -1.7406389713287354,
+      "logps/chosen": -211.9698944091797,
+      "logps/ref_chosen": -226.790771484375,
+      "logps/ref_rejected": -231.8648223876953,
+      "logps/rejected": -219.543212890625,
+      "loss": 0.4945,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.1482090801000595,
+      "rewards/margins": 0.024992961436510086,
+      "rewards/rejected": 0.12321610748767853,
       "step": 130
     },
     {
       "epoch": 0.5770772107934812,
-      "grad_norm": 1.054673194885254,
-      "learning_rate": 4.512251721523659e-07,
-      "logits/chosen": -2.005443811416626,
-      "logits/rejected": -2.0154759883880615,
-      "logps/chosen": -0.7493831515312195,
-      "logps/ref_chosen": -0.7740285992622375,
-      "logps/ref_rejected": -0.8138446807861328,
-      "logps/rejected": -0.851770281791687,
-      "loss": 0.4847,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": 0.06161379814147949,
-      "rewards/margins": 0.15642789006233215,
-      "rewards/rejected": -0.09481407701969147,
+      "grad_norm": 0.5537051558494568,
+      "learning_rate": 2.2561258607618294e-07,
+      "logits/chosen": -1.8008477687835693,
+      "logits/rejected": -1.8080832958221436,
+      "logps/chosen": -234.68893432617188,
+      "logps/ref_chosen": -247.26119995117188,
+      "logps/ref_rejected": -241.82345581054688,
+      "logps/rejected": -231.585693359375,
+      "loss": 0.4949,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.12572243809700012,
+      "rewards/margins": 0.023345012217760086,
+      "rewards/rejected": 0.10237739980220795,
       "step": 135
     },
     {
       "epoch": 0.5984504408228694,
-      "grad_norm": 1.669247031211853,
-      "learning_rate": 4.139984308181708e-07,
-      "logits/chosen": -1.9584558010101318,
-      "logits/rejected": -1.8885042667388916,
-      "logps/chosen": -0.7844404578208923,
-      "logps/ref_chosen": -0.8231161236763,
-      "logps/ref_rejected": -0.83356112241745,
-      "logps/rejected": -0.8187839388847351,
-      "loss": 0.481,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.09668895602226257,
-      "rewards/margins": 0.0597461462020874,
-      "rewards/rejected": 0.03694281354546547,
+      "grad_norm": 0.5528976321220398,
+      "learning_rate": 2.069992154090854e-07,
+      "logits/chosen": -1.775397539138794,
+      "logits/rejected": -1.6931631565093994,
+      "logps/chosen": -219.74072265625,
+      "logps/ref_chosen": -230.71826171875,
+      "logps/ref_rejected": -227.7001953125,
+      "logps/rejected": -218.38241577148438,
+      "loss": 0.495,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.10977540910243988,
+      "rewards/margins": 0.01659761555492878,
+      "rewards/rejected": 0.09317778795957565,
       "step": 140
     },
     {
       "epoch": 0.6198236708522575,
-      "grad_norm": 1.0050734281539917,
-      "learning_rate": 3.772572564296004e-07,
-      "logits/chosen": -1.8883240222930908,
-      "logits/rejected": -1.8240330219268799,
-      "logps/chosen": -0.7662582397460938,
-      "logps/ref_chosen": -0.8861669301986694,
-      "logps/ref_rejected": -0.924543023109436,
-      "logps/rejected": -0.8522801399230957,
-      "loss": 0.4785,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.29977160692214966,
-      "rewards/margins": 0.11911455541849136,
-      "rewards/rejected": 0.18065707385540009,
+      "grad_norm": 0.5473525524139404,
+      "learning_rate": 1.886286282148002e-07,
+      "logits/chosen": -1.7711913585662842,
+      "logits/rejected": -1.7026926279067993,
+      "logps/chosen": -195.3854217529297,
+      "logps/ref_chosen": -208.07254028320312,
+      "logps/ref_rejected": -210.4279022216797,
+      "logps/rejected": -199.79165649414062,
+      "loss": 0.4946,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.12687113881111145,
+      "rewards/margins": 0.020508771762251854,
+      "rewards/rejected": 0.10636236518621445,
       "step": 145
     },
     {
       "epoch": 0.6411969008816457,
-      "grad_norm": 0.9123177528381348,
-      "learning_rate": 3.412090905484337e-07,
-      "logits/chosen": -1.9726581573486328,
-      "logits/rejected": -1.9151983261108398,
-      "logps/chosen": -0.7370959520339966,
-      "logps/ref_chosen": -0.866258978843689,
-      "logps/ref_rejected": -0.8657606840133667,
-      "logps/rejected": -0.8145822286605835,
-      "loss": 0.4817,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.3229079246520996,
-      "rewards/margins": 0.19496168196201324,
-      "rewards/rejected": 0.12794628739356995,
+      "grad_norm": 0.5966719388961792,
+      "learning_rate": 1.7060454527421686e-07,
+      "logits/chosen": -1.8688771724700928,
+      "logits/rejected": -1.810694932937622,
+      "logps/chosen": -211.9062042236328,
+      "logps/ref_chosen": -224.8968505859375,
+      "logps/ref_rejected": -226.1548309326172,
+      "logps/rejected": -215.7084503173828,
+      "loss": 0.4943,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.12990659475326538,
+      "rewards/margins": 0.02544253133237362,
+      "rewards/rejected": 0.10446406900882721,
       "step": 150
     },
     {
       "epoch": 0.6625701309110339,
-      "grad_norm": 0.7277486324310303,
-      "learning_rate": 3.060574619936075e-07,
-      "logits/chosen": -1.8861163854599,
-      "logits/rejected": -1.9004647731781006,
-      "logps/chosen": -0.7181011438369751,
-      "logps/ref_chosen": -0.8273455500602722,
-      "logps/ref_rejected": -0.8894198536872864,
-      "logps/rejected": -0.8557901382446289,
-      "loss": 0.4828,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.27311110496520996,
-      "rewards/margins": 0.18903681635856628,
-      "rewards/rejected": 0.08407425880432129,
+      "grad_norm": 0.5334843993186951,
+      "learning_rate": 1.5302873099680374e-07,
+      "logits/chosen": -1.786595344543457,
+      "logits/rejected": -1.7971456050872803,
+      "logps/chosen": -225.0083465576172,
+      "logps/ref_chosen": -237.4626922607422,
+      "logps/ref_rejected": -234.39547729492188,
+      "logps/rejected": -223.2943572998047,
+      "loss": 0.4955,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.12454362213611603,
+      "rewards/margins": 0.013532285578548908,
+      "rewards/rejected": 0.1110113263130188,
       "step": 155
     },
     {
       "epoch": 0.6839433609404221,
-      "grad_norm": 1.0556169748306274,
-      "learning_rate": 2.720008377125682e-07,
-      "logits/chosen": -2.1498093605041504,
-      "logits/rejected": -2.08402419090271,
-      "logps/chosen": -0.7090437412261963,
-      "logps/ref_chosen": -0.8103801012039185,
-      "logps/ref_rejected": -0.8715206980705261,
-      "logps/rejected": -0.7982600927352905,
-      "loss": 0.4815,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.25334107875823975,
-      "rewards/margins": 0.0701896995306015,
-      "rewards/rejected": 0.18315134942531586,
+      "grad_norm": 0.5639063715934753,
+      "learning_rate": 1.360004188562841e-07,
+      "logits/chosen": -2.0527145862579346,
+      "logits/rejected": -1.9811140298843384,
+      "logps/chosen": -217.0570068359375,
+      "logps/ref_chosen": -231.03369140625,
+      "logps/ref_rejected": -232.6383819580078,
+      "logps/rejected": -220.0625457763672,
+      "loss": 0.4952,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.1397666186094284,
+      "rewards/margins": 0.014008410274982452,
+      "rewards/rejected": 0.12575821578502655,
       "step": 160
     },
     {
       "epoch": 0.7053165909698104,
-      "grad_norm": 0.9068896770477295,
-      "learning_rate": 2.3923150223207173e-07,
-      "logits/chosen": -1.9448268413543701,
-      "logits/rejected": -1.904314637184143,
-      "logps/chosen": -0.7294695377349854,
-      "logps/ref_chosen": -0.8327474594116211,
-      "logps/ref_rejected": -0.9134753346443176,
-      "logps/rejected": -0.8374517560005188,
-      "loss": 0.4781,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.25819462537765503,
-      "rewards/margins": 0.06813579052686691,
-      "rewards/rejected": 0.1900588572025299,
+      "grad_norm": 0.5417853593826294,
+      "learning_rate": 1.1961575111603586e-07,
+      "logits/chosen": -1.8371235132217407,
+      "logits/rejected": -1.7954612970352173,
+      "logps/chosen": -220.7694854736328,
+      "logps/ref_chosen": -234.5041046142578,
+      "logps/ref_rejected": -235.61181640625,
+      "logps/rejected": -224.56640625,
+      "loss": 0.4944,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.1373465657234192,
+      "rewards/margins": 0.026892542839050293,
+      "rewards/rejected": 0.1104540079832077,
       "step": 165
     },
     {
       "epoch": 0.7266898209991985,
-      "grad_norm": 0.9222660660743713,
-      "learning_rate": 2.0793447201508286e-07,
-      "logits/chosen": -1.9369819164276123,
-      "logits/rejected": -1.9469242095947266,
-      "logps/chosen": -0.6714679002761841,
-      "logps/ref_chosen": -0.7705163359642029,
-      "logps/ref_rejected": -0.8333786129951477,
-      "logps/rejected": -0.7605674862861633,
-      "loss": 0.4845,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.24762126803398132,
-      "rewards/margins": 0.06559363007545471,
-      "rewards/rejected": 0.18202762305736542,
+      "grad_norm": 0.565830409526825,
+      "learning_rate": 1.0396723600754143e-07,
+      "logits/chosen": -1.8288425207138062,
+      "logits/rejected": -1.83499276638031,
+      "logps/chosen": -213.2861785888672,
+      "logps/ref_chosen": -227.1809844970703,
+      "logps/ref_rejected": -230.8953094482422,
+      "logps/rejected": -218.4414520263672,
+      "loss": 0.4954,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.13894793391227722,
+      "rewards/margins": 0.014409348368644714,
+      "rewards/rejected": 0.12453857809305191,
       "step": 170
     },
     {
       "epoch": 0.7480630510285867,
-      "grad_norm": 0.7805910110473633,
-      "learning_rate": 1.7828645085333644e-07,
-      "logits/chosen": -1.9515256881713867,
-      "logits/rejected": -1.9057369232177734,
-      "logps/chosen": -0.7692245244979858,
-      "logps/ref_chosen": -0.8767744302749634,
-      "logps/ref_rejected": -0.8914516568183899,
-      "logps/rejected": -0.8368504643440247,
-      "loss": 0.4772,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.2688748240470886,
-      "rewards/margins": 0.13237187266349792,
-      "rewards/rejected": 0.1365029364824295,
+      "grad_norm": 0.5855058431625366,
+      "learning_rate": 8.914322542666822e-08,
+      "logits/chosen": -1.8145122528076172,
+      "logits/rejected": -1.7646887302398682,
+      "logps/chosen": -212.070068359375,
+      "logps/ref_chosen": -224.17794799804688,
+      "logps/ref_rejected": -225.526123046875,
+      "logps/rejected": -214.7656707763672,
+      "loss": 0.4947,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.12107895314693451,
+      "rewards/margins": 0.013474419713020325,
+      "rewards/rejected": 0.10760452598333359,
       "step": 175
     },
     {
       "epoch": 0.7694362810579749,
-      "grad_norm": 0.8946753740310669,
-      "learning_rate": 1.5045483219344385e-07,
-      "logits/chosen": -2.008927583694458,
-      "logits/rejected": -1.9914191961288452,
-      "logps/chosen": -0.7557133436203003,
-      "logps/ref_chosen": -0.8390854597091675,
-      "logps/ref_rejected": -0.8787837028503418,
-      "logps/rejected": -0.853840708732605,
-      "loss": 0.479,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.2084302008152008,
-      "rewards/margins": 0.14607290923595428,
-      "rewards/rejected": 0.062357254326343536,
+      "grad_norm": 0.6223751902580261,
+      "learning_rate": 7.522741609672193e-08,
+      "logits/chosen": -1.8675405979156494,
+      "logits/rejected": -1.8476943969726562,
+      "logps/chosen": -216.3776092529297,
+      "logps/ref_chosen": -230.77182006835938,
+      "logps/ref_rejected": -227.00619506835938,
+      "logps/rejected": -214.32931518554688,
+      "loss": 0.4945,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.1439422070980072,
+      "rewards/margins": 0.017173700034618378,
+      "rewards/rejected": 0.12676851451396942,
       "step": 180
     },
     {
       "epoch": 0.7908095110873631,
-      "grad_norm": 1.0278949737548828,
-      "learning_rate": 1.2459675402943288e-07,
-      "logits/chosen": -2.0313353538513184,
-      "logits/rejected": -1.9398431777954102,
-      "logps/chosen": -0.7753286361694336,
-      "logps/ref_chosen": -0.8660305142402649,
-      "logps/ref_rejected": -0.8604307174682617,
-      "logps/rejected": -0.7984707951545715,
-      "loss": 0.4765,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.22675485908985138,
-      "rewards/margins": 0.07185501605272293,
-      "rewards/rejected": 0.15489983558654785,
+      "grad_norm": 0.5778200030326843,
+      "learning_rate": 6.229837701471644e-08,
+      "logits/chosen": -1.9124794006347656,
+      "logits/rejected": -1.8135532140731812,
+      "logps/chosen": -216.97702026367188,
+      "logps/ref_chosen": -229.8362274169922,
+      "logps/ref_rejected": -233.65390014648438,
+      "logps/rejected": -222.93417358398438,
+      "loss": 0.4945,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.1285921037197113,
+      "rewards/margins": 0.021394768729805946,
+      "rewards/rejected": 0.10719730705022812,
       "step": 185
     },
     {
       "epoch": 0.8121827411167513,
-      "grad_norm": 0.8263163566589355,
-      "learning_rate": 1.0085821169782199e-07,
-      "logits/chosen": -2.0978431701660156,
-      "logits/rejected": -2.055168628692627,
-      "logps/chosen": -0.7470763921737671,
-      "logps/ref_chosen": -0.8603025674819946,
-      "logps/ref_rejected": -0.9167188405990601,
-      "logps/rejected": -0.8492987751960754,
-      "loss": 0.4776,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.2830653786659241,
-      "rewards/margins": 0.11451487243175507,
-      "rewards/rejected": 0.1685505211353302,
+      "grad_norm": 0.5558175444602966,
+      "learning_rate": 5.0429105848910996e-08,
+      "logits/chosen": -1.9621855020523071,
+      "logits/rejected": -1.9175077676773071,
+      "logps/chosen": -215.39450073242188,
+      "logps/ref_chosen": -229.72836303710938,
+      "logps/ref_rejected": -233.65237426757812,
+      "logps/rejected": -222.21798706054688,
+      "loss": 0.4937,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.14333853125572205,
+      "rewards/margins": 0.028994807973504066,
+      "rewards/rejected": 0.11434372514486313,
       "step": 190
     },
     {
       "epoch": 0.8335559711461394,
-      "grad_norm": 0.9788782596588135,
-      "learning_rate": 7.937323358440934e-08,
-      "logits/chosen": -2.13584041595459,
-      "logits/rejected": -2.0809855461120605,
-      "logps/chosen": -0.7344987988471985,
-      "logps/ref_chosen": -0.8153272867202759,
-      "logps/ref_rejected": -0.8524506688117981,
-      "logps/rejected": -0.8127390742301941,
-      "loss": 0.475,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.20207130908966064,
-      "rewards/margins": 0.10279206931591034,
-      "rewards/rejected": 0.09927921742200851,
+      "grad_norm": 0.5308636426925659,
+      "learning_rate": 3.968661679220467e-08,
+      "logits/chosen": -1.971208930015564,
+      "logits/rejected": -1.9112732410430908,
+      "logps/chosen": -210.79598999023438,
+      "logps/ref_chosen": -224.2023468017578,
+      "logps/ref_rejected": -224.3248748779297,
+      "logps/rejected": -212.8175811767578,
+      "loss": 0.4932,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.1340634524822235,
+      "rewards/margins": 0.018990488722920418,
+      "rewards/rejected": 0.11507296562194824,
       "step": 195
     },
     {
       "epoch": 0.8549292011755276,
-      "grad_norm": 1.1459167003631592,
-      "learning_rate": 6.026312439675551e-08,
-      "logits/chosen": -1.9530729055404663,
-      "logits/rejected": -1.8388773202896118,
-      "logps/chosen": -0.7524019479751587,
-      "logps/ref_chosen": -0.832992672920227,
-      "logps/ref_rejected": -0.8378564715385437,
-      "logps/rejected": -0.7900499701499939,
-      "loss": 0.4783,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.20147652924060822,
-      "rewards/margins": 0.08196047693490982,
-      "rewards/rejected": 0.11951601505279541,
+      "grad_norm": 0.615912675857544,
+      "learning_rate": 3.013156219837776e-08,
+      "logits/chosen": -1.7899879217147827,
+      "logits/rejected": -1.6696176528930664,
+      "logps/chosen": -215.92288208007812,
+      "logps/ref_chosen": -228.88381958007812,
+      "logps/ref_rejected": -231.0583953857422,
+      "logps/rejected": -220.5959930419922,
+      "loss": 0.4932,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.12960924208164215,
+      "rewards/margins": 0.024985069409012794,
+      "rewards/rejected": 0.1046241745352745,
       "step": 200
     },
     {
       "epoch": 0.8763024312049158,
-      "grad_norm": 1.2217403650283813,
-      "learning_rate": 4.3635780274861864e-08,
-      "logits/chosen": -1.9760059118270874,
-      "logits/rejected": -1.881757378578186,
-      "logps/chosen": -0.754524827003479,
-      "logps/ref_chosen": -0.8361877202987671,
-      "logps/ref_rejected": -0.8636215329170227,
-      "logps/rejected": -0.8430477380752563,
-      "loss": 0.4761,
+      "grad_norm": 0.590220034122467,
+      "learning_rate": 2.1817890137430932e-08,
+      "logits/chosen": -1.81471848487854,
+      "logits/rejected": -1.714023232460022,
+      "logps/chosen": -205.69888305664062,
+      "logps/ref_chosen": -221.30752563476562,
+      "logps/ref_rejected": -224.98486328125,
+      "logps/rejected": -211.78884887695312,
+      "loss": 0.4937,
       "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.20415742695331573,
-      "rewards/margins": 0.15272292494773865,
-      "rewards/rejected": 0.05143451690673828,
+      "rewards/chosen": 0.15608620643615723,
+      "rewards/margins": 0.024126073345541954,
+      "rewards/rejected": 0.13196012377738953,
       "step": 205
     },
     {
       "epoch": 0.897675661234304,
-      "grad_norm": 0.9312232732772827,
-      "learning_rate": 2.958507960694784e-08,
-      "logits/chosen": -1.9826923608779907,
-      "logits/rejected": -1.963322401046753,
-      "logps/chosen": -0.7218400239944458,
-      "logps/ref_chosen": -0.775153636932373,
-      "logps/ref_rejected": -0.82710200548172,
-      "logps/rejected": -0.8231655359268188,
-      "loss": 0.4756,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.13328400254249573,
-      "rewards/margins": 0.12344253063201904,
-      "rewards/rejected": 0.009841480292379856,
+      "grad_norm": 0.5369106531143188,
+      "learning_rate": 1.479253980347392e-08,
+      "logits/chosen": -1.8037662506103516,
+      "logits/rejected": -1.7787643671035767,
+      "logps/chosen": -225.9608612060547,
+      "logps/ref_chosen": -241.4657440185547,
+      "logps/ref_rejected": -241.3707733154297,
+      "logps/rejected": -228.4087371826172,
+      "loss": 0.4931,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.15504886209964752,
+      "rewards/margins": 0.025428583845496178,
+      "rewards/rejected": 0.1296202689409256,
       "step": 210
     },
     {
       "epoch": 0.9190488912636923,
-      "grad_norm": 0.9846080541610718,
-      "learning_rate": 1.8190352989793322e-08,
-      "logits/chosen": -1.9855095148086548,
-      "logits/rejected": -1.907576560974121,
-      "logps/chosen": -0.7199736833572388,
-      "logps/ref_chosen": -0.803063690662384,
-      "logps/ref_rejected": -0.8516971468925476,
-      "logps/rejected": -0.8365123867988586,
-      "loss": 0.4776,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.2077248990535736,
-      "rewards/margins": 0.16976311802864075,
-      "rewards/rejected": 0.03796178475022316,
+      "grad_norm": 0.5737273097038269,
+      "learning_rate": 9.095176494896661e-09,
+      "logits/chosen": -1.8023388385772705,
+      "logits/rejected": -1.7160924673080444,
+      "logps/chosen": -218.32034301757812,
+      "logps/ref_chosen": -231.6717071533203,
+      "logps/ref_rejected": -236.741943359375,
+      "logps/rejected": -225.2128448486328,
+      "loss": 0.4933,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.13351376354694366,
+      "rewards/margins": 0.018222931772470474,
+      "rewards/rejected": 0.11529083549976349,
       "step": 215
     },
     {
       "epoch": 0.9404221212930804,
-      "grad_norm": 0.7335969805717468,
-      "learning_rate": 9.515935326265378e-09,
-      "logits/chosen": -2.0024361610412598,
-      "logits/rejected": -1.9636704921722412,
-      "logps/chosen": -0.7521845698356628,
-      "logps/ref_chosen": -0.8253963589668274,
-      "logps/ref_rejected": -0.849533200263977,
-      "logps/rejected": -0.8330303430557251,
-      "loss": 0.4757,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.1830292046070099,
-      "rewards/margins": 0.14177197217941284,
-      "rewards/rejected": 0.04125722497701645,
+      "grad_norm": 0.6087775826454163,
+      "learning_rate": 4.757967663132689e-09,
+      "logits/chosen": -1.833620309829712,
+      "logits/rejected": -1.7870299816131592,
+      "logps/chosen": -221.86032104492188,
+      "logps/ref_chosen": -236.0878448486328,
+      "logps/ref_rejected": -230.54141235351562,
+      "logps/rejected": -218.8464813232422,
+      "loss": 0.4935,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.14227530360221863,
+      "rewards/margins": 0.025325754657387733,
+      "rewards/rejected": 0.11694953590631485,
       "step": 220
     },
     {
       "epoch": 0.9617953513224686,
-      "grad_norm": 1.1543854475021362,
-      "learning_rate": 3.6108025888958447e-09,
-      "logits/chosen": -1.9360746145248413,
-      "logits/rejected": -1.911627173423767,
-      "logps/chosen": -0.7084980010986328,
-      "logps/ref_chosen": -0.7970255613327026,
-      "logps/ref_rejected": -0.8132475018501282,
-      "logps/rejected": -0.7666771411895752,
-      "loss": 0.4765,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.22131893038749695,
-      "rewards/margins": 0.10489317029714584,
-      "rewards/rejected": 0.1164257749915123,
+      "grad_norm": 0.6274195909500122,
+      "learning_rate": 1.8054012944479224e-09,
+      "logits/chosen": -1.7650978565216064,
+      "logits/rejected": -1.7383601665496826,
+      "logps/chosen": -231.64111328125,
+      "logps/ref_chosen": -244.44155883789062,
+      "logps/ref_rejected": -240.8953094482422,
+      "logps/rejected": -230.3839874267578,
+      "loss": 0.4932,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.12800416350364685,
+      "rewards/margins": 0.022890925407409668,
+      "rewards/rejected": 0.10511324554681778,
       "step": 225
     },
     {
       "epoch": 0.9831685813518568,
-      "grad_norm": 1.1417807340621948,
-      "learning_rate": 5.082953003528456e-10,
-      "logits/chosen": -2.013756513595581,
-      "logits/rejected": -2.0383520126342773,
-      "logps/chosen": -0.8249381184577942,
-      "logps/ref_chosen": -0.8979822993278503,
-      "logps/ref_rejected": -0.9172071218490601,
-      "logps/rejected": -0.9182927012443542,
-      "loss": 0.4774,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.1826106607913971,
-      "rewards/margins": 0.18532457947731018,
-      "rewards/rejected": -0.0027139366138726473,
+      "grad_norm": 0.5350868105888367,
+      "learning_rate": 2.541476501764228e-10,
+      "logits/chosen": -1.8503191471099854,
+      "logits/rejected": -1.878313660621643,
+      "logps/chosen": -206.16665649414062,
+      "logps/ref_chosen": -219.6629638671875,
+      "logps/ref_rejected": -212.42172241210938,
+      "logps/rejected": -200.54551696777344,
+      "loss": 0.494,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.13496311008930206,
+      "rewards/margins": 0.016201000660657883,
+      "rewards/rejected": 0.11876209825277328,
       "step": 230
     },
     {
       "epoch": 0.9959925193694897,
       "step": 233,
       "total_flos": 0.0,
-      "train_loss": 0.4860667634931245,
-      "train_runtime": 16529.3176,
-      "train_samples_per_second": 3.622,
+      "train_loss": 0.49642937480124283,
+      "train_runtime": 16410.2083,
+      "train_samples_per_second": 3.649,
       "train_steps_per_second": 0.014
     }
   ],