diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,3183 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9886363636363638,
+  "eval_steps": 50.0,
+  "global_step": 210,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00946969696969697,
+      "grad_norm": 29.0056470375542,
+      "learning_rate": 4.761904761904761e-09,
+      "logits/chosen": -1.2287073135375977,
+      "logits/rejected": -1.226149559020996,
+      "logps/chosen": -298.41802978515625,
+      "logps/rejected": -125.05058288574219,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01893939393939394,
+      "grad_norm": 29.598707271309717,
+      "learning_rate": 9.523809523809522e-09,
+      "logits/chosen": -1.1785930395126343,
+      "logits/rejected": -1.2503385543823242,
+      "logps/chosen": -327.542236328125,
+      "logps/rejected": -152.739990234375,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2
+    },
+    {
+      "epoch": 0.028409090909090908,
+      "grad_norm": 30.815773047267687,
+      "learning_rate": 1.4285714285714284e-08,
+      "logits/chosen": -1.2728688716888428,
+      "logits/rejected": -1.2958910465240479,
+      "logps/chosen": -162.8279266357422,
+      "logps/rejected": -92.91324615478516,
+      "loss": 0.6939,
+      "rewards/accuracies": 0.46666663885116577,
+      "rewards/chosen": 0.0023424469400197268,
+      "rewards/margins": 0.005515547469258308,
+      "rewards/rejected": -0.003173100296407938,
+      "step": 3
+    },
+    {
+      "epoch": 0.03787878787878788,
+      "grad_norm": 30.410758788816498,
+      "learning_rate": 1.9047619047619045e-08,
+      "logits/chosen": -1.3308671712875366,
+      "logits/rejected": -1.4291285276412964,
+      "logps/chosen": -361.55377197265625,
+      "logps/rejected": -185.06399536132812,
+      "loss": 0.6944,
+      "rewards/accuracies": 0.2666666805744171,
+      "rewards/chosen": -0.0036378882359713316,
+      "rewards/margins": -0.005622108932584524,
+      "rewards/rejected": 0.001984220463782549,
+      "step": 4
+    },
+    {
+      "epoch": 0.04734848484848485,
+      "grad_norm": 33.72650012517962,
+      "learning_rate": 2.3809523809523807e-08,
+      "logits/chosen": -1.2832980155944824,
+      "logits/rejected": -1.2578494548797607,
+      "logps/chosen": -311.66046142578125,
+      "logps/rejected": -113.93507385253906,
+      "loss": 0.6944,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": 0.0004738619609270245,
+      "rewards/margins": 0.000339136749971658,
+      "rewards/rejected": 0.0001347253128187731,
+      "step": 5
+    },
+    {
+      "epoch": 0.056818181818181816,
+      "grad_norm": 33.54755632608565,
+      "learning_rate": 2.857142857142857e-08,
+      "logits/chosen": -1.2438254356384277,
+      "logits/rejected": -1.3154207468032837,
+      "logps/chosen": -237.88046264648438,
+      "logps/rejected": -144.86170959472656,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0015382947167381644,
+      "rewards/margins": 0.0072678267024457455,
+      "rewards/rejected": -0.005729531869292259,
+      "step": 6
+    },
+    {
+      "epoch": 0.06628787878787878,
+      "grad_norm": 32.506024543201846,
+      "learning_rate": 3.333333333333333e-08,
+      "logits/chosen": -1.1219017505645752,
+      "logits/rejected": -1.3356969356536865,
+      "logps/chosen": -377.674072265625,
+      "logps/rejected": -208.0950164794922,
+      "loss": 0.6946,
+      "rewards/accuracies": 0.46666663885116577,
+      "rewards/chosen": 0.003615258727222681,
+      "rewards/margins": 0.004607487935572863,
+      "rewards/rejected": -0.0009922297904267907,
+      "step": 7
+    },
+    {
+      "epoch": 0.07575757575757576,
+      "grad_norm": 26.869719248200415,
+      "learning_rate": 3.809523809523809e-08,
+      "logits/chosen": -1.2021812200546265,
+      "logits/rejected": -1.2994741201400757,
+      "logps/chosen": -223.2887420654297,
+      "logps/rejected": -210.76904296875,
+      "loss": 0.6941,
+      "rewards/accuracies": 0.46666663885116577,
+      "rewards/chosen": -0.0010813308181241155,
+      "rewards/margins": -0.0014500105753540993,
+      "rewards/rejected": 0.00036868025199510157,
+      "step": 8
+    },
+    {
+      "epoch": 0.08522727272727272,
+      "grad_norm": 28.851260723123424,
+      "learning_rate": 4.285714285714285e-08,
+      "logits/chosen": -1.2686998844146729,
+      "logits/rejected": -1.2887214422225952,
+      "logps/chosen": -414.627685546875,
+      "logps/rejected": -157.76890563964844,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.3333333432674408,
+      "rewards/chosen": -0.008543035015463829,
+      "rewards/margins": -0.008067367598414421,
+      "rewards/rejected": -0.00047566817374899983,
+      "step": 9
+    },
+    {
+      "epoch": 0.0946969696969697,
+      "grad_norm": 29.908833040399635,
+      "learning_rate": 4.7619047619047613e-08,
+      "logits/chosen": -1.2252384424209595,
+      "logits/rejected": -1.4116103649139404,
+      "logps/chosen": -379.18719482421875,
+      "logps/rejected": -134.59957885742188,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": 0.003142343368381262,
+      "rewards/margins": 0.004382309503853321,
+      "rewards/rejected": -0.0012399652041494846,
+      "step": 10
+    },
+    {
+      "epoch": 0.10416666666666667,
+      "grad_norm": 34.95508468311557,
+      "learning_rate": 5.238095238095238e-08,
+      "logits/chosen": -1.2304970026016235,
+      "logits/rejected": -1.4173061847686768,
+      "logps/chosen": -274.32415771484375,
+      "logps/rejected": -143.79139709472656,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.5333333015441895,
+      "rewards/chosen": 0.001248523243702948,
+      "rewards/margins": 0.00037327370955608785,
+      "rewards/rejected": 0.0008752490393817425,
+      "step": 11
+    },
+    {
+      "epoch": 0.11363636363636363,
+      "grad_norm": 33.36383008667387,
+      "learning_rate": 5.714285714285714e-08,
+      "logits/chosen": -1.2315393686294556,
+      "logits/rejected": -1.3125958442687988,
+      "logps/chosen": -414.57220458984375,
+      "logps/rejected": -175.58706665039062,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.009653424844145775,
+      "rewards/margins": 0.005845812149345875,
+      "rewards/rejected": 0.0038076122291386127,
+      "step": 12
+    },
+    {
+      "epoch": 0.12310606060606061,
+      "grad_norm": 33.31390804979405,
+      "learning_rate": 6.19047619047619e-08,
+      "logits/chosen": -1.226459264755249,
+      "logits/rejected": -1.333916425704956,
+      "logps/chosen": -220.2379913330078,
+      "logps/rejected": -109.96634674072266,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": 0.003358738962560892,
+      "rewards/margins": 0.0006390871712937951,
+      "rewards/rejected": 0.002719651209190488,
+      "step": 13
+    },
+    {
+      "epoch": 0.13257575757575757,
+      "grad_norm": 30.861328252470425,
+      "learning_rate": 6.666666666666665e-08,
+      "logits/chosen": -1.2634985446929932,
+      "logits/rejected": -1.4346576929092407,
+      "logps/chosen": -280.9840087890625,
+      "logps/rejected": -140.47079467773438,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": -0.0012296470813453197,
+      "rewards/margins": -0.0024472051300108433,
+      "rewards/rejected": 0.0012175582814961672,
+      "step": 14
+    },
+    {
+      "epoch": 0.14204545454545456,
+      "grad_norm": 29.301187205022977,
+      "learning_rate": 7.142857142857142e-08,
+      "logits/chosen": -1.2319475412368774,
+      "logits/rejected": -1.3931156396865845,
+      "logps/chosen": -279.740234375,
+      "logps/rejected": -63.6292724609375,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": 0.005358916707336903,
+      "rewards/margins": 0.00306467292830348,
+      "rewards/rejected": 0.002294243313372135,
+      "step": 15
+    },
+    {
+      "epoch": 0.15151515151515152,
+      "grad_norm": 33.40165234179044,
+      "learning_rate": 7.619047619047618e-08,
+      "logits/chosen": -1.2616360187530518,
+      "logits/rejected": -1.2523891925811768,
+      "logps/chosen": -248.74118041992188,
+      "logps/rejected": -139.3803253173828,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.00993367563933134,
+      "rewards/margins": 0.0073961131274700165,
+      "rewards/rejected": 0.0025375632103532553,
+      "step": 16
+    },
+    {
+      "epoch": 0.16098484848484848,
+      "grad_norm": 31.850407532445676,
+      "learning_rate": 8.095238095238095e-08,
+      "logits/chosen": -1.300764560699463,
+      "logits/rejected": -1.4897656440734863,
+      "logps/chosen": -199.72067260742188,
+      "logps/rejected": -117.5413818359375,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": 0.005169999785721302,
+      "rewards/margins": 0.004817936569452286,
+      "rewards/rejected": 0.0003520629834383726,
+      "step": 17
+    },
+    {
+      "epoch": 0.17045454545454544,
+      "grad_norm": 31.778736512108093,
+      "learning_rate": 8.57142857142857e-08,
+      "logits/chosen": -1.2460119724273682,
+      "logits/rejected": -1.3352010250091553,
+      "logps/chosen": -430.80096435546875,
+      "logps/rejected": -209.36929321289062,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.022602636367082596,
+      "rewards/margins": 0.02024615928530693,
+      "rewards/rejected": 0.002356476616114378,
+      "step": 18
+    },
+    {
+      "epoch": 0.17992424242424243,
+      "grad_norm": 31.888287525403957,
+      "learning_rate": 9.047619047619047e-08,
+      "logits/chosen": -1.2491956949234009,
+      "logits/rejected": -1.2977077960968018,
+      "logps/chosen": -226.297607421875,
+      "logps/rejected": -110.85136413574219,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.5333333015441895,
+      "rewards/chosen": 0.0118248350918293,
+      "rewards/margins": 0.008896713145077229,
+      "rewards/rejected": 0.0029281210154294968,
+      "step": 19
+    },
+    {
+      "epoch": 0.1893939393939394,
+      "grad_norm": 30.890781750471852,
+      "learning_rate": 9.523809523809523e-08,
+      "logits/chosen": -1.2209726572036743,
+      "logits/rejected": -1.4927175045013428,
+      "logps/chosen": -337.1795959472656,
+      "logps/rejected": -123.1536865234375,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": 0.019501259550452232,
+      "rewards/margins": 0.021095363423228264,
+      "rewards/rejected": -0.001594100845977664,
+      "step": 20
+    },
+    {
+      "epoch": 0.19886363636363635,
+      "grad_norm": 29.49005689794317,
+      "learning_rate": 1e-07,
+      "logits/chosen": -1.2870718240737915,
+      "logits/rejected": -1.3624413013458252,
+      "logps/chosen": -226.2629852294922,
+      "logps/rejected": -94.34611511230469,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": 0.011245525442063808,
+      "rewards/margins": 0.0074931420385837555,
+      "rewards/rejected": 0.0037523836363106966,
+      "step": 21
+    },
+    {
+      "epoch": 0.20833333333333334,
+      "grad_norm": 35.528523903361425,
+      "learning_rate": 9.947089947089947e-08,
+      "logits/chosen": -1.149263858795166,
+      "logits/rejected": -1.1778504848480225,
+      "logps/chosen": -522.42626953125,
+      "logps/rejected": -187.16226196289062,
+      "loss": 0.6755,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.05816841125488281,
+      "rewards/margins": 0.0636506900191307,
+      "rewards/rejected": -0.005482281092554331,
+      "step": 22
+    },
+    {
+      "epoch": 0.2178030303030303,
+      "grad_norm": 30.463057051689077,
+      "learning_rate": 9.894179894179894e-08,
+      "logits/chosen": -1.2491704225540161,
+      "logits/rejected": -1.4007856845855713,
+      "logps/chosen": -221.8249053955078,
+      "logps/rejected": -94.48908233642578,
+      "loss": 0.6733,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": 0.03221073001623154,
+      "rewards/margins": 0.02945198491215706,
+      "rewards/rejected": 0.0027587474323809147,
+      "step": 23
+    },
+    {
+      "epoch": 0.22727272727272727,
+      "grad_norm": 32.66406919328301,
+      "learning_rate": 9.84126984126984e-08,
+      "logits/chosen": -1.213392734527588,
+      "logits/rejected": -1.2315367460250854,
+      "logps/chosen": -484.7833557128906,
+      "logps/rejected": -179.44985961914062,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.07118237763643265,
+      "rewards/margins": 0.07119101285934448,
+      "rewards/rejected": -8.63661989569664e-06,
+      "step": 24
+    },
+    {
+      "epoch": 0.23674242424242425,
+      "grad_norm": 27.949898051091324,
+      "learning_rate": 9.788359788359788e-08,
+      "logits/chosen": -1.2136789560317993,
+      "logits/rejected": -1.3596720695495605,
+      "logps/chosen": -315.8499755859375,
+      "logps/rejected": -146.63519287109375,
+      "loss": 0.6695,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.05525461584329605,
+      "rewards/margins": 0.05192388966679573,
+      "rewards/rejected": 0.003330723149701953,
+      "step": 25
+    },
+    {
+      "epoch": 0.24621212121212122,
+      "grad_norm": 31.30737207115018,
+      "learning_rate": 9.735449735449735e-08,
+      "logits/chosen": -1.326267957687378,
+      "logits/rejected": -1.3485438823699951,
+      "logps/chosen": -318.2695007324219,
+      "logps/rejected": -93.18646240234375,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.06859429180622101,
+      "rewards/margins": 0.06752636283636093,
+      "rewards/rejected": 0.0010679200058802962,
+      "step": 26
+    },
+    {
+      "epoch": 0.2556818181818182,
+      "grad_norm": 27.163805337499927,
+      "learning_rate": 9.682539682539682e-08,
+      "logits/chosen": -1.3347467184066772,
+      "logits/rejected": -1.39474618434906,
+      "logps/chosen": -252.51211547851562,
+      "logps/rejected": -130.60934448242188,
+      "loss": 0.6685,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": 0.04859574884176254,
+      "rewards/margins": 0.04069235175848007,
+      "rewards/rejected": 0.007903401739895344,
+      "step": 27
+    },
+    {
+      "epoch": 0.26515151515151514,
+      "grad_norm": 26.747663753533093,
+      "learning_rate": 9.629629629629629e-08,
+      "logits/chosen": -1.1969085931777954,
+      "logits/rejected": -1.3106380701065063,
+      "logps/chosen": -282.17694091796875,
+      "logps/rejected": -131.32461547851562,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.053814828395843506,
+      "rewards/margins": 0.05295072868466377,
+      "rewards/rejected": 0.0008641040185466409,
+      "step": 28
+    },
+    {
+      "epoch": 0.2746212121212121,
+      "grad_norm": 28.110282404080507,
+      "learning_rate": 9.576719576719576e-08,
+      "logits/chosen": -1.205107569694519,
+      "logits/rejected": -1.3407138586044312,
+      "logps/chosen": -262.7644958496094,
+      "logps/rejected": -134.28187561035156,
+      "loss": 0.6658,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.05242891237139702,
+      "rewards/margins": 0.05327294394373894,
+      "rewards/rejected": -0.0008440311066806316,
+      "step": 29
+    },
+    {
+      "epoch": 0.2840909090909091,
+      "grad_norm": 24.744190890187145,
+      "learning_rate": 9.523809523809523e-08,
+      "logits/chosen": -1.2627979516983032,
+      "logits/rejected": -1.3917124271392822,
+      "logps/chosen": -238.56619262695312,
+      "logps/rejected": -120.55537414550781,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.04753679037094116,
+      "rewards/margins": 0.04130668193101883,
+      "rewards/rejected": 0.006230102851986885,
+      "step": 30
+    },
+    {
+      "epoch": 0.2935606060606061,
+      "grad_norm": 27.968644068426777,
+      "learning_rate": 9.470899470899471e-08,
+      "logits/chosen": -1.3139160871505737,
+      "logits/rejected": -1.4469630718231201,
+      "logps/chosen": -368.27374267578125,
+      "logps/rejected": -130.02462768554688,
+      "loss": 0.6626,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": 0.07182395458221436,
+      "rewards/margins": 0.07256630808115005,
+      "rewards/rejected": -0.000742347736377269,
+      "step": 31
+    },
+    {
+      "epoch": 0.30303030303030304,
+      "grad_norm": 26.902166854256873,
+      "learning_rate": 9.417989417989417e-08,
+      "logits/chosen": -1.2979252338409424,
+      "logits/rejected": -1.2534756660461426,
+      "logps/chosen": -361.4090576171875,
+      "logps/rejected": -205.6992645263672,
+      "loss": 0.6487,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.10240572690963745,
+      "rewards/margins": 0.09547950327396393,
+      "rewards/rejected": 0.006926238536834717,
+      "step": 32
+    },
+    {
+      "epoch": 0.3125,
+      "grad_norm": 24.41388186700755,
+      "learning_rate": 9.365079365079365e-08,
+      "logits/chosen": -1.2682493925094604,
+      "logits/rejected": -1.3996570110321045,
+      "logps/chosen": -301.144775390625,
+      "logps/rejected": -178.94155883789062,
+      "loss": 0.6366,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": 0.1342059224843979,
+      "rewards/margins": 0.09688643366098404,
+      "rewards/rejected": 0.03731948137283325,
+      "step": 33
+    },
+    {
+      "epoch": 0.32196969696969696,
+      "grad_norm": 23.025986498306636,
+      "learning_rate": 9.312169312169311e-08,
+      "logits/chosen": -1.2429295778274536,
+      "logits/rejected": -1.3105340003967285,
+      "logps/chosen": -333.86077880859375,
+      "logps/rejected": -116.7296371459961,
+      "loss": 0.6333,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.1280738264322281,
+      "rewards/margins": 0.13231658935546875,
+      "rewards/rejected": -0.004242760129272938,
+      "step": 34
+    },
+    {
+      "epoch": 0.3314393939393939,
+      "grad_norm": 24.498334455805622,
+      "learning_rate": 9.259259259259259e-08,
+      "logits/chosen": -1.3265695571899414,
+      "logits/rejected": -1.3860602378845215,
+      "logps/chosen": -299.00750732421875,
+      "logps/rejected": -140.71592712402344,
+      "loss": 0.6286,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": 0.11404500156641006,
+      "rewards/margins": 0.11178859323263168,
+      "rewards/rejected": 0.002256409265100956,
+      "step": 35
+    },
+    {
+      "epoch": 0.3409090909090909,
+      "grad_norm": 24.884921186077833,
+      "learning_rate": 9.206349206349205e-08,
+      "logits/chosen": -1.2984504699707031,
+      "logits/rejected": -1.3744697570800781,
+      "logps/chosen": -285.7705078125,
+      "logps/rejected": -122.30088806152344,
+      "loss": 0.6168,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.15358424186706543,
+      "rewards/margins": 0.15173228085041046,
+      "rewards/rejected": 0.001851982669904828,
+      "step": 36
+    },
+    {
+      "epoch": 0.3503787878787879,
+      "grad_norm": 23.82555156631703,
+      "learning_rate": 9.153439153439153e-08,
+      "logits/chosen": -1.2476222515106201,
+      "logits/rejected": -1.3255198001861572,
+      "logps/chosen": -366.74029541015625,
+      "logps/rejected": -130.18682861328125,
+      "loss": 0.6238,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.1664523035287857,
+      "rewards/margins": 0.15406028926372528,
+      "rewards/rejected": 0.012392010539770126,
+      "step": 37
+    },
+    {
+      "epoch": 0.35984848484848486,
+      "grad_norm": 24.07477880888288,
+      "learning_rate": 9.1005291005291e-08,
+      "logits/chosen": -1.335256814956665,
+      "logits/rejected": -1.4410061836242676,
+      "logps/chosen": -219.09616088867188,
+      "logps/rejected": -99.69376373291016,
+      "loss": 0.6137,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": 0.1590159386396408,
+      "rewards/margins": 0.1581113636493683,
+      "rewards/rejected": 0.0009045897168107331,
+      "step": 38
+    },
+    {
+      "epoch": 0.3693181818181818,
+      "grad_norm": 23.16275958406235,
+      "learning_rate": 9.047619047619047e-08,
+      "logits/chosen": -1.2550255060195923,
+      "logits/rejected": -1.3113772869110107,
+      "logps/chosen": -370.3097839355469,
+      "logps/rejected": -170.9385986328125,
+      "loss": 0.6112,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23174059391021729,
+      "rewards/margins": 0.23885612189769745,
+      "rewards/rejected": -0.00711551308631897,
+      "step": 39
+    },
+    {
+      "epoch": 0.3787878787878788,
+      "grad_norm": 21.543309745960354,
+      "learning_rate": 8.994708994708994e-08,
+      "logits/chosen": -1.3221899271011353,
+      "logits/rejected": -1.4326074123382568,
+      "logps/chosen": -290.75933837890625,
+      "logps/rejected": -167.4901123046875,
+      "loss": 0.6159,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": 0.13931059837341309,
+      "rewards/margins": 0.13773180544376373,
+      "rewards/rejected": 0.0015788152813911438,
+      "step": 40
+    },
+    {
+      "epoch": 0.38825757575757575,
+      "grad_norm": 22.418812006239037,
+      "learning_rate": 8.941798941798941e-08,
+      "logits/chosen": -1.2849162817001343,
+      "logits/rejected": -1.3343697786331177,
+      "logps/chosen": -369.89764404296875,
+      "logps/rejected": -133.05355834960938,
+      "loss": 0.6093,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.19583266973495483,
+      "rewards/margins": 0.19024832546710968,
+      "rewards/rejected": 0.005584346130490303,
+      "step": 41
+    },
+    {
+      "epoch": 0.3977272727272727,
+      "grad_norm": 25.71519469395891,
+      "learning_rate": 8.888888888888888e-08,
+      "logits/chosen": -1.286842703819275,
+      "logits/rejected": -1.3505761623382568,
+      "logps/chosen": -447.53778076171875,
+      "logps/rejected": -138.85006713867188,
+      "loss": 0.5864,
+      "rewards/accuracies": 0.9333332777023315,
+      "rewards/chosen": 0.31389278173446655,
+      "rewards/margins": 0.3263290822505951,
+      "rewards/rejected": -0.01243629027158022,
+      "step": 42
+    },
+    {
+      "epoch": 0.4071969696969697,
+      "grad_norm": 19.24790527585849,
+      "learning_rate": 8.835978835978835e-08,
+      "logits/chosen": -1.3817297220230103,
+      "logits/rejected": -1.4520673751831055,
+      "logps/chosen": -279.92059326171875,
+      "logps/rejected": -121.86234283447266,
+      "loss": 0.6187,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.15583908557891846,
+      "rewards/margins": 0.15511645376682281,
+      "rewards/rejected": 0.0007226344314403832,
+      "step": 43
+    },
+    {
+      "epoch": 0.4166666666666667,
+      "grad_norm": 22.13389198587296,
+      "learning_rate": 8.783068783068782e-08,
+      "logits/chosen": -1.3588807582855225,
+      "logits/rejected": -1.3591158390045166,
+      "logps/chosen": -430.5394592285156,
+      "logps/rejected": -167.34222412109375,
+      "loss": 0.5983,
+      "rewards/accuracies": 0.9333332777023315,
+      "rewards/chosen": 0.22864654660224915,
+      "rewards/margins": 0.19965213537216187,
+      "rewards/rejected": 0.028994422405958176,
+      "step": 44
+    },
+    {
+      "epoch": 0.42613636363636365,
+      "grad_norm": 19.709066122231608,
+      "learning_rate": 8.730158730158729e-08,
+      "logits/chosen": -1.319620132446289,
+      "logits/rejected": -1.3754303455352783,
+      "logps/chosen": -301.44891357421875,
+      "logps/rejected": -139.29481506347656,
+      "loss": 0.6036,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.22895975410938263,
+      "rewards/margins": 0.213044673204422,
+      "rewards/rejected": 0.015915077179670334,
+      "step": 45
+    },
+    {
+      "epoch": 0.4356060606060606,
+      "grad_norm": 20.432654588017233,
+      "learning_rate": 8.677248677248676e-08,
+      "logits/chosen": -1.314937710762024,
+      "logits/rejected": -1.3968394994735718,
+      "logps/chosen": -240.7213897705078,
+      "logps/rejected": -84.15922546386719,
+      "loss": 0.6013,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.15969710052013397,
+      "rewards/margins": 0.16978248953819275,
+      "rewards/rejected": -0.010085375979542732,
+      "step": 46
+    },
+    {
+      "epoch": 0.44507575757575757,
+      "grad_norm": 17.302724108599822,
+      "learning_rate": 8.624338624338625e-08,
+      "logits/chosen": -1.3449150323867798,
+      "logits/rejected": -1.4744312763214111,
+      "logps/chosen": -345.28387451171875,
+      "logps/rejected": -119.1164321899414,
+      "loss": 0.6147,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.1876756250858307,
+      "rewards/margins": 0.18070700764656067,
+      "rewards/rejected": 0.006968595087528229,
+      "step": 47
+    },
+    {
+      "epoch": 0.45454545454545453,
+      "grad_norm": 18.038050767324112,
+      "learning_rate": 8.57142857142857e-08,
+      "logits/chosen": -1.3185852766036987,
+      "logits/rejected": -1.4122602939605713,
+      "logps/chosen": -212.75405883789062,
+      "logps/rejected": -140.37655639648438,
+      "loss": 0.6088,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.1278039515018463,
+      "rewards/margins": 0.04179360717535019,
+      "rewards/rejected": 0.08601033687591553,
+      "step": 48
+    },
+    {
+      "epoch": 0.4640151515151515,
+      "grad_norm": 22.986861694117593,
+      "learning_rate": 8.518518518518519e-08,
+      "logits/chosen": -1.2588977813720703,
+      "logits/rejected": -1.4479458332061768,
+      "logps/chosen": -338.35858154296875,
+      "logps/rejected": -192.6273956298828,
+      "loss": 0.5795,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.22073189914226532,
+      "rewards/margins": 0.24155020713806152,
+      "rewards/rejected": -0.02081829309463501,
+      "step": 49
+    },
+    {
+      "epoch": 0.4734848484848485,
+      "grad_norm": 19.82460905470704,
+      "learning_rate": 8.465608465608464e-08,
+      "logits/chosen": -1.2549892663955688,
+      "logits/rejected": -1.4687315225601196,
+      "logps/chosen": -310.2621154785156,
+      "logps/rejected": -137.7465057373047,
+      "loss": 0.5936,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.2339044064283371,
+      "rewards/margins": 0.22999358177185059,
+      "rewards/rejected": 0.003910848870873451,
+      "step": 50
+    },
+    {
+      "epoch": 0.48295454545454547,
+      "grad_norm": 19.652707095228752,
+      "learning_rate": 8.412698412698413e-08,
+      "logits/chosen": -1.397512674331665,
+      "logits/rejected": -1.4295556545257568,
+      "logps/chosen": -350.259033203125,
+      "logps/rejected": -154.9229278564453,
+      "loss": 0.5959,
+      "rewards/accuracies": 0.73333340883255,
+      "rewards/chosen": 0.2114483118057251,
+      "rewards/margins": 0.1936836689710617,
+      "rewards/rejected": 0.017764627933502197,
+      "step": 51
+    },
+    {
+      "epoch": 0.49242424242424243,
+      "grad_norm": 18.932470131303603,
+      "learning_rate": 8.359788359788358e-08,
+      "logits/chosen": -1.3295793533325195,
+      "logits/rejected": -1.4178295135498047,
+      "logps/chosen": -228.3561248779297,
+      "logps/rejected": -152.6548309326172,
+      "loss": 0.5968,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.17612284421920776,
+      "rewards/margins": 0.1884087324142456,
+      "rewards/rejected": -0.012285866774618626,
+      "step": 52
+    },
+    {
+      "epoch": 0.5018939393939394,
+      "grad_norm": 22.56023512380409,
+      "learning_rate": 8.306878306878307e-08,
+      "logits/chosen": -1.329941749572754,
+      "logits/rejected": -1.437350869178772,
+      "logps/chosen": -272.7892150878906,
+      "logps/rejected": -130.10751342773438,
+      "loss": 0.562,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.2890825569629669,
+      "rewards/margins": 0.3040001392364502,
+      "rewards/rejected": -0.014917601831257343,
+      "step": 53
+    },
+    {
+      "epoch": 0.5113636363636364,
+      "grad_norm": 16.32960137441355,
+      "learning_rate": 8.253968253968254e-08,
+      "logits/chosen": -1.4222466945648193,
+      "logits/rejected": -1.6047979593276978,
+      "logps/chosen": -327.20196533203125,
+      "logps/rejected": -94.08998107910156,
+      "loss": 0.5619,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.3207431137561798,
+      "rewards/margins": 0.362914502620697,
+      "rewards/rejected": -0.0421714186668396,
+      "step": 54
+    },
+    {
+      "epoch": 0.5208333333333334,
+      "grad_norm": 16.970648675586034,
+      "learning_rate": 8.201058201058201e-08,
+      "logits/chosen": -1.4205853939056396,
+      "logits/rejected": -1.4251158237457275,
+      "logps/chosen": -283.05108642578125,
+      "logps/rejected": -105.49992370605469,
+      "loss": 0.547,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.31010201573371887,
+      "rewards/margins": 0.3413071930408478,
+      "rewards/rejected": -0.031205186620354652,
+      "step": 55
+    },
+    {
+      "epoch": 0.5303030303030303,
+      "grad_norm": 13.112771276876163,
+      "learning_rate": 8.148148148148148e-08,
+      "logits/chosen": -1.3618199825286865,
+      "logits/rejected": -1.506471037864685,
+      "logps/chosen": -345.26434326171875,
+      "logps/rejected": -164.83676147460938,
+      "loss": 0.5754,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.41369715332984924,
+      "rewards/margins": 0.42479389905929565,
+      "rewards/rejected": -0.011096751317381859,
+      "step": 56
+    },
+    {
+      "epoch": 0.5397727272727273,
+      "grad_norm": 15.00178283487482,
+      "learning_rate": 8.095238095238095e-08,
+      "logits/chosen": -1.3916102647781372,
+      "logits/rejected": -1.4226547479629517,
+      "logps/chosen": -266.93389892578125,
+      "logps/rejected": -123.35113525390625,
+      "loss": 0.5448,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.2940821647644043,
+      "rewards/margins": 0.32441002130508423,
+      "rewards/rejected": -0.030327826738357544,
+      "step": 57
+    },
+    {
+      "epoch": 0.5492424242424242,
+      "grad_norm": 13.894167304668269,
+      "learning_rate": 8.042328042328042e-08,
+      "logits/chosen": -1.4024507999420166,
+      "logits/rejected": -1.499112844467163,
+      "logps/chosen": -299.7988586425781,
+      "logps/rejected": -212.5677032470703,
+      "loss": 0.5271,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.4066668450832367,
+      "rewards/margins": 0.4445282816886902,
+      "rewards/rejected": -0.03786151856184006,
+      "step": 58
+    },
+    {
+      "epoch": 0.5587121212121212,
+      "grad_norm": 13.686152249765941,
+      "learning_rate": 7.989417989417989e-08,
+      "logits/chosen": -1.4456459283828735,
+      "logits/rejected": -1.4960581064224243,
+      "logps/chosen": -218.0140380859375,
+      "logps/rejected": -91.45022583007812,
+      "loss": 0.5359,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.23951879143714905,
+      "rewards/margins": 0.2641497552394867,
+      "rewards/rejected": -0.02463097684085369,
+      "step": 59
+    },
+    {
+      "epoch": 0.5681818181818182,
+      "grad_norm": 13.294977223020641,
+      "learning_rate": 7.936507936507936e-08,
+      "logits/chosen": -1.446040391921997,
+      "logits/rejected": -1.4068689346313477,
+      "logps/chosen": -271.784912109375,
+      "logps/rejected": -104.23944091796875,
+      "loss": 0.5248,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.33832794427871704,
+      "rewards/margins": 0.3952670991420746,
+      "rewards/rejected": -0.05693921446800232,
+      "step": 60
+    },
+    {
+      "epoch": 0.5776515151515151,
+      "grad_norm": 14.213258268574368,
+      "learning_rate": 7.883597883597883e-08,
+      "logits/chosen": -1.525424599647522,
+      "logits/rejected": -1.5090603828430176,
+      "logps/chosen": -307.25115966796875,
+      "logps/rejected": -123.07181549072266,
+      "loss": 0.5072,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.2689701318740845,
+      "rewards/margins": 0.3598218560218811,
+      "rewards/rejected": -0.09085171669721603,
+      "step": 61
+    },
+    {
+      "epoch": 0.5871212121212122,
+      "grad_norm": 11.942720090111948,
+      "learning_rate": 7.83068783068783e-08,
+      "logits/chosen": -1.341205358505249,
+      "logits/rejected": -1.4334052801132202,
+      "logps/chosen": -235.2104034423828,
+      "logps/rejected": -95.51054382324219,
+      "loss": 0.512,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.2976926863193512,
+      "rewards/margins": 0.3563759922981262,
+      "rewards/rejected": -0.05868334323167801,
+      "step": 62
+    },
+    {
+      "epoch": 0.5965909090909091,
+      "grad_norm": 14.098400355821441,
+      "learning_rate": 7.777777777777778e-08,
+      "logits/chosen": -1.5624217987060547,
+      "logits/rejected": -1.550048828125,
+      "logps/chosen": -395.7930603027344,
+      "logps/rejected": -119.40779113769531,
+      "loss": 0.487,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.40899619460105896,
+      "rewards/margins": 0.47930893301963806,
+      "rewards/rejected": -0.07031276077032089,
+      "step": 63
+    },
+    {
+      "epoch": 0.6060606060606061,
+      "grad_norm": 11.1041564672598,
+      "learning_rate": 7.724867724867724e-08,
+      "logits/chosen": -1.4298704862594604,
+      "logits/rejected": -1.354206919670105,
+      "logps/chosen": -312.83697509765625,
+      "logps/rejected": -110.90681457519531,
+      "loss": 0.5224,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.32718759775161743,
+      "rewards/margins": 0.3963702321052551,
+      "rewards/rejected": -0.06918264925479889,
+      "step": 64
+    },
+    {
+      "epoch": 0.615530303030303,
+      "grad_norm": 13.357200204611026,
+      "learning_rate": 7.671957671957672e-08,
+      "logits/chosen": -1.423771619796753,
+      "logits/rejected": -1.495290756225586,
+      "logps/chosen": -341.0536193847656,
+      "logps/rejected": -195.5428466796875,
+      "loss": 0.4798,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.47605428099632263,
+      "rewards/margins": 0.5310575366020203,
+      "rewards/rejected": -0.055003322660923004,
+      "step": 65
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 13.546486727992278,
+      "learning_rate": 7.619047619047618e-08,
+      "logits/chosen": -1.5028284788131714,
+      "logits/rejected": -1.4564129114151,
+      "logps/chosen": -285.58917236328125,
+      "logps/rejected": -131.5936279296875,
+      "loss": 0.4836,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.36600998044013977,
+      "rewards/margins": 0.4425935745239258,
+      "rewards/rejected": -0.0765836089849472,
+      "step": 66
+    },
+    {
+      "epoch": 0.634469696969697,
+      "grad_norm": 12.596961700455271,
+      "learning_rate": 7.566137566137566e-08,
+      "logits/chosen": -1.4141170978546143,
+      "logits/rejected": -1.4340078830718994,
+      "logps/chosen": -261.4342041015625,
+      "logps/rejected": -95.14923858642578,
+      "loss": 0.4883,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.44839000701904297,
+      "rewards/margins": 0.5747641324996948,
+      "rewards/rejected": -0.12637418508529663,
+      "step": 67
+    },
+    {
+      "epoch": 0.6439393939393939,
+      "grad_norm": 12.384595380406164,
+      "learning_rate": 7.513227513227512e-08,
+      "logits/chosen": -1.4482860565185547,
+      "logits/rejected": -1.5673211812973022,
+      "logps/chosen": -237.4990234375,
+      "logps/rejected": -84.68649291992188,
+      "loss": 0.5,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.37629473209381104,
+      "rewards/margins": 0.42733898758888245,
+      "rewards/rejected": -0.05104423686861992,
+      "step": 68
+    },
+    {
+      "epoch": 0.6534090909090909,
+      "grad_norm": 11.573197865338908,
+      "learning_rate": 7.46031746031746e-08,
+      "logits/chosen": -1.4828598499298096,
+      "logits/rejected": -1.5483115911483765,
+      "logps/chosen": -294.4015197753906,
+      "logps/rejected": -105.67552185058594,
+      "loss": 0.4914,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.42700010538101196,
+      "rewards/margins": 0.5032913684844971,
+      "rewards/rejected": -0.07629130035638809,
+      "step": 69
+    },
+    {
+      "epoch": 0.6628787878787878,
+      "grad_norm": 11.526501769285554,
+      "learning_rate": 7.407407407407407e-08,
+      "logits/chosen": -1.3778324127197266,
+      "logits/rejected": -1.3970375061035156,
+      "logps/chosen": -356.0700988769531,
+      "logps/rejected": -174.7995147705078,
+      "loss": 0.4855,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.5114725232124329,
+      "rewards/margins": 0.5073350071907043,
+      "rewards/rejected": 0.004137503914535046,
+      "step": 70
+    },
+    {
+      "epoch": 0.6723484848484849,
+      "grad_norm": 11.786777529592367,
+      "learning_rate": 7.354497354497354e-08,
+      "logits/chosen": -1.4470304250717163,
+      "logits/rejected": -1.5260601043701172,
+      "logps/chosen": -257.9070739746094,
+      "logps/rejected": -82.23036193847656,
+      "loss": 0.5138,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.3575395941734314,
+      "rewards/margins": 0.4714414179325104,
+      "rewards/rejected": -0.11390187591314316,
+      "step": 71
+    },
+    {
+      "epoch": 0.6818181818181818,
+      "grad_norm": 13.911478948655224,
+      "learning_rate": 7.301587301587301e-08,
+      "logits/chosen": -1.4967644214630127,
+      "logits/rejected": -1.379651427268982,
+      "logps/chosen": -456.3885803222656,
+      "logps/rejected": -181.92654418945312,
+      "loss": 0.4686,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6715552806854248,
+      "rewards/margins": 0.8583688735961914,
+      "rewards/rejected": -0.18681350350379944,
+      "step": 72
+    },
+    {
+      "epoch": 0.6912878787878788,
+      "grad_norm": 11.631616954008805,
+      "learning_rate": 7.248677248677248e-08,
+      "logits/chosen": -1.4026285409927368,
+      "logits/rejected": -1.4543156623840332,
+      "logps/chosen": -337.62078857421875,
+      "logps/rejected": -139.98934936523438,
+      "loss": 0.5001,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.5019295811653137,
+      "rewards/margins": 0.5479447245597839,
+      "rewards/rejected": -0.04601515084505081,
+      "step": 73
+    },
+    {
+      "epoch": 0.7007575757575758,
+      "grad_norm": 9.936203845833878,
+      "learning_rate": 7.195767195767195e-08,
+      "logits/chosen": -1.5050134658813477,
+      "logits/rejected": -1.5326780080795288,
+      "logps/chosen": -387.0226135253906,
+      "logps/rejected": -181.61231994628906,
+      "loss": 0.5285,
+      "rewards/accuracies": 0.9333332777023315,
+      "rewards/chosen": 0.47982874512672424,
+      "rewards/margins": 0.5457210540771484,
+      "rewards/rejected": -0.0658923089504242,
+      "step": 74
+    },
+    {
+      "epoch": 0.7102272727272727,
+      "grad_norm": 12.184941282416535,
+      "learning_rate": 7.142857142857142e-08,
+      "logits/chosen": -1.4602888822555542,
+      "logits/rejected": -1.4935940504074097,
+      "logps/chosen": -471.65643310546875,
+      "logps/rejected": -181.0192108154297,
+      "loss": 0.4831,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.696118175983429,
+      "rewards/margins": 0.8435744047164917,
+      "rewards/rejected": -0.1474563181400299,
+      "step": 75
+    },
+    {
+      "epoch": 0.7196969696969697,
+      "grad_norm": 11.235163839695879,
+      "learning_rate": 7.08994708994709e-08,
+      "logits/chosen": -1.5082679986953735,
+      "logits/rejected": -1.5974117517471313,
+      "logps/chosen": -399.4970703125,
+      "logps/rejected": -201.58975219726562,
+      "loss": 0.4768,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6427886486053467,
+      "rewards/margins": 0.7292692065238953,
+      "rewards/rejected": -0.08648059517145157,
+      "step": 76
+    },
+    {
+      "epoch": 0.7291666666666666,
+      "grad_norm": 12.9574754516344,
+      "learning_rate": 7.037037037037036e-08,
+      "logits/chosen": -1.4920450448989868,
+      "logits/rejected": -1.4171216487884521,
+      "logps/chosen": -332.895263671875,
+      "logps/rejected": -176.00601196289062,
+      "loss": 0.4783,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.4683101773262024,
+      "rewards/margins": 0.5503407716751099,
+      "rewards/rejected": -0.08203061670064926,
+      "step": 77
+    },
+    {
+      "epoch": 0.7386363636363636,
+      "grad_norm": 11.558597299124692,
+      "learning_rate": 6.984126984126983e-08,
+      "logits/chosen": -1.4621433019638062,
+      "logits/rejected": -1.5110585689544678,
+      "logps/chosen": -361.80889892578125,
+      "logps/rejected": -157.5341339111328,
+      "loss": 0.4715,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.555467963218689,
+      "rewards/margins": 0.682567834854126,
+      "rewards/rejected": -0.12709984183311462,
+      "step": 78
+    },
+    {
+      "epoch": 0.7481060606060606,
+      "grad_norm": 13.548458950264191,
+      "learning_rate": 6.931216931216932e-08,
+      "logits/chosen": -1.523284673690796,
+      "logits/rejected": -1.4904415607452393,
+      "logps/chosen": -309.17816162109375,
+      "logps/rejected": -115.497802734375,
+      "loss": 0.493,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.4258233904838562,
+      "rewards/margins": 0.4799923896789551,
+      "rewards/rejected": -0.05416899174451828,
+      "step": 79
+    },
+    {
+      "epoch": 0.7575757575757576,
+      "grad_norm": 10.624056172874939,
+      "learning_rate": 6.878306878306877e-08,
+      "logits/chosen": -1.4721736907958984,
+      "logits/rejected": -1.556168794631958,
+      "logps/chosen": -349.3233337402344,
+      "logps/rejected": -119.07454681396484,
+      "loss": 0.4971,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.4824630320072174,
+      "rewards/margins": 0.6270266771316528,
+      "rewards/rejected": -0.1445636749267578,
+      "step": 80
+    },
+    {
+      "epoch": 0.7670454545454546,
+      "grad_norm": 10.877663952011163,
+      "learning_rate": 6.825396825396826e-08,
+      "logits/chosen": -1.448540210723877,
+      "logits/rejected": -1.5003968477249146,
+      "logps/chosen": -341.19549560546875,
+      "logps/rejected": -161.4232635498047,
+      "loss": 0.4997,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.512890636920929,
+      "rewards/margins": 0.5583428144454956,
+      "rewards/rejected": -0.04545217379927635,
+      "step": 81
+    },
+    {
+      "epoch": 0.7765151515151515,
+      "grad_norm": 13.655792524642862,
+      "learning_rate": 6.772486772486771e-08,
+      "logits/chosen": -1.4932104349136353,
+      "logits/rejected": -1.4824309349060059,
+      "logps/chosen": -377.6290588378906,
+      "logps/rejected": -145.4966583251953,
+      "loss": 0.4506,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5544673204421997,
+      "rewards/margins": 0.7561152577400208,
+      "rewards/rejected": -0.20164790749549866,
+      "step": 82
+    },
+    {
+      "epoch": 0.7859848484848485,
+      "grad_norm": 13.094215069002113,
+      "learning_rate": 6.71957671957672e-08,
+      "logits/chosen": -1.5782554149627686,
+      "logits/rejected": -1.660269021987915,
+      "logps/chosen": -349.7228088378906,
+      "logps/rejected": -182.11880493164062,
+      "loss": 0.4818,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.6241387128829956,
+      "rewards/margins": 0.7656179666519165,
+      "rewards/rejected": -0.1414792239665985,
+      "step": 83
+    },
+    {
+      "epoch": 0.7954545454545454,
+      "grad_norm": 12.779977757002404,
+      "learning_rate": 6.666666666666665e-08,
+      "logits/chosen": -1.5646089315414429,
+      "logits/rejected": -1.5739343166351318,
+      "logps/chosen": -311.35760498046875,
+      "logps/rejected": -131.74862670898438,
+      "loss": 0.4655,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5058180093765259,
+      "rewards/margins": 0.6643168926239014,
+      "rewards/rejected": -0.1584988683462143,
+      "step": 84
+    },
+    {
+      "epoch": 0.8049242424242424,
+      "grad_norm": 10.185073249389749,
+      "learning_rate": 6.613756613756614e-08,
+      "logits/chosen": -1.4740874767303467,
+      "logits/rejected": -1.4708284139633179,
+      "logps/chosen": -319.00286865234375,
+      "logps/rejected": -152.86892700195312,
+      "loss": 0.4822,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5287100076675415,
+      "rewards/margins": 0.6535229682922363,
+      "rewards/rejected": -0.1248130053281784,
+      "step": 85
+    },
+    {
+      "epoch": 0.8143939393939394,
+      "grad_norm": 11.203293253753769,
+      "learning_rate": 6.560846560846561e-08,
+      "logits/chosen": -1.5367662906646729,
+      "logits/rejected": -1.4478954076766968,
+      "logps/chosen": -201.14149475097656,
+      "logps/rejected": -86.86661529541016,
+      "loss": 0.4671,
+      "rewards/accuracies": 0.9333332777023315,
+      "rewards/chosen": 0.37096017599105835,
+      "rewards/margins": 0.45440879464149475,
+      "rewards/rejected": -0.0834486186504364,
+      "step": 86
+    },
+    {
+      "epoch": 0.8238636363636364,
+      "grad_norm": 10.604116644121472,
+      "learning_rate": 6.507936507936508e-08,
+      "logits/chosen": -1.4273946285247803,
+      "logits/rejected": -1.4703240394592285,
+      "logps/chosen": -301.97882080078125,
+      "logps/rejected": -157.4232635498047,
+      "loss": 0.4514,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.624559223651886,
+      "rewards/margins": 0.7758625745773315,
+      "rewards/rejected": -0.15130344033241272,
+      "step": 87
+    },
+    {
+      "epoch": 0.8333333333333334,
+      "grad_norm": 9.972392236357932,
+      "learning_rate": 6.455026455026455e-08,
+      "logits/chosen": -1.482578158378601,
+      "logits/rejected": -1.4413843154907227,
+      "logps/chosen": -254.91702270507812,
+      "logps/rejected": -123.97944641113281,
+      "loss": 0.4721,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.48685789108276367,
+      "rewards/margins": 0.6410495638847351,
+      "rewards/rejected": -0.1541917622089386,
+      "step": 88
+    },
+    {
+      "epoch": 0.8428030303030303,
+      "grad_norm": 10.523325958099775,
+      "learning_rate": 6.402116402116402e-08,
+      "logits/chosen": -1.480163335800171,
+      "logits/rejected": -1.5355768203735352,
+      "logps/chosen": -304.09197998046875,
+      "logps/rejected": -142.2599639892578,
+      "loss": 0.4744,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.5564759969711304,
+      "rewards/margins": 0.6676918864250183,
+      "rewards/rejected": -0.11121580749750137,
+      "step": 89
+    },
+    {
+      "epoch": 0.8522727272727273,
+      "grad_norm": 9.519106187436845,
+      "learning_rate": 6.349206349206349e-08,
+      "logits/chosen": -1.5268163681030273,
+      "logits/rejected": -1.5646402835845947,
+      "logps/chosen": -286.10540771484375,
+      "logps/rejected": -157.7232208251953,
+      "loss": 0.4714,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.4824487268924713,
+      "rewards/margins": 0.5735751986503601,
+      "rewards/rejected": -0.09112647920846939,
+      "step": 90
+    },
+    {
+      "epoch": 0.8617424242424242,
+      "grad_norm": 9.51590024478788,
+      "learning_rate": 6.296296296296296e-08,
+      "logits/chosen": -1.5810682773590088,
+      "logits/rejected": -1.561959981918335,
+      "logps/chosen": -292.758056640625,
+      "logps/rejected": -114.890625,
+      "loss": 0.5022,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.47544798254966736,
+      "rewards/margins": 0.5433242321014404,
+      "rewards/rejected": -0.06787623465061188,
+      "step": 91
+    },
+    {
+      "epoch": 0.8712121212121212,
+      "grad_norm": 9.702200423985945,
+      "learning_rate": 6.243386243386243e-08,
+      "logits/chosen": -1.4159653186798096,
+      "logits/rejected": -1.5273025035858154,
+      "logps/chosen": -310.64093017578125,
+      "logps/rejected": -160.01473999023438,
+      "loss": 0.4457,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.5551848411560059,
+      "rewards/margins": 0.6525019407272339,
+      "rewards/rejected": -0.097317174077034,
+      "step": 92
+    },
+    {
+      "epoch": 0.8806818181818182,
+      "grad_norm": 12.532628428602317,
+      "learning_rate": 6.19047619047619e-08,
+      "logits/chosen": -1.4812260866165161,
+      "logits/rejected": -1.5698903799057007,
+      "logps/chosen": -313.69659423828125,
+      "logps/rejected": -125.74027252197266,
+      "loss": 0.4221,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.512529730796814,
+      "rewards/margins": 0.7795059084892273,
+      "rewards/rejected": -0.26697611808776855,
+      "step": 93
+    },
+    {
+      "epoch": 0.8901515151515151,
+      "grad_norm": 9.937234934609606,
+      "learning_rate": 6.137566137566137e-08,
+      "logits/chosen": -1.4872493743896484,
+      "logits/rejected": -1.4849263429641724,
+      "logps/chosen": -306.2376708984375,
+      "logps/rejected": -151.22828674316406,
+      "loss": 0.4843,
+      "rewards/accuracies": 0.9333332777023315,
+      "rewards/chosen": 0.49157848954200745,
+      "rewards/margins": 0.6916278004646301,
+      "rewards/rejected": -0.2000492364168167,
+      "step": 94
+    },
+    {
+      "epoch": 0.8996212121212122,
+      "grad_norm": 10.966223203578922,
+      "learning_rate": 6.084656084656084e-08,
+      "logits/chosen": -1.5097806453704834,
+      "logits/rejected": -1.548575520515442,
+      "logps/chosen": -248.2743682861328,
+      "logps/rejected": -143.56008911132812,
+      "loss": 0.4619,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.29616254568099976,
+      "rewards/margins": 0.44729262590408325,
+      "rewards/rejected": -0.1511300802230835,
+      "step": 95
+    },
+    {
+      "epoch": 0.9090909090909091,
+      "grad_norm": 10.290146618878584,
+      "learning_rate": 6.031746031746031e-08,
+      "logits/chosen": -1.3948135375976562,
+      "logits/rejected": -1.4724781513214111,
+      "logps/chosen": -200.97560119628906,
+      "logps/rejected": -59.250823974609375,
+      "loss": 0.4559,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.31829187273979187,
+      "rewards/margins": 0.4826883375644684,
+      "rewards/rejected": -0.1643964797258377,
+      "step": 96
+    },
+    {
+      "epoch": 0.9185606060606061,
+      "grad_norm": 10.023113393783985,
+      "learning_rate": 5.978835978835979e-08,
+      "logits/chosen": -1.4924052953720093,
+      "logits/rejected": -1.481069803237915,
+      "logps/chosen": -210.6460418701172,
+      "logps/rejected": -101.57307434082031,
+      "loss": 0.4809,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.3884206712245941,
+      "rewards/margins": 0.5861561298370361,
+      "rewards/rejected": -0.19773544371128082,
+      "step": 97
+    },
+    {
+      "epoch": 0.928030303030303,
+      "grad_norm": 9.340197405706663,
+      "learning_rate": 5.925925925925925e-08,
+      "logits/chosen": -1.5180368423461914,
+      "logits/rejected": -1.4653794765472412,
+      "logps/chosen": -321.68426513671875,
+      "logps/rejected": -144.31082153320312,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.9333332777023315,
+      "rewards/chosen": 0.4283772110939026,
+      "rewards/margins": 0.8070729970932007,
+      "rewards/rejected": -0.3786957859992981,
+      "step": 98
+    },
+    {
+      "epoch": 0.9375,
+      "grad_norm": 9.85547123624337,
+      "learning_rate": 5.873015873015873e-08,
+      "logits/chosen": -1.5044419765472412,
+      "logits/rejected": -1.5305202007293701,
+      "logps/chosen": -289.38946533203125,
+      "logps/rejected": -109.14179992675781,
+      "loss": 0.4682,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.4926362931728363,
+      "rewards/margins": 0.6987640261650085,
+      "rewards/rejected": -0.20612768828868866,
+      "step": 99
+    },
+    {
+      "epoch": 0.946969696969697,
+      "grad_norm": 10.445428542153785,
+      "learning_rate": 5.82010582010582e-08,
+      "logits/chosen": -1.4615473747253418,
+      "logits/rejected": -1.507441520690918,
+      "logps/chosen": -233.7719268798828,
+      "logps/rejected": -87.18647766113281,
+      "loss": 0.4687,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.3351677358150482,
+      "rewards/margins": 0.5392307043075562,
+      "rewards/rejected": -0.20406293869018555,
+      "step": 100
+    },
+    {
+      "epoch": 0.9564393939393939,
+      "grad_norm": 10.068872305516752,
+      "learning_rate": 5.7671957671957674e-08,
+      "logits/chosen": -1.4435585737228394,
+      "logits/rejected": -1.592414379119873,
+      "logps/chosen": -288.0964660644531,
+      "logps/rejected": -245.06857299804688,
+      "loss": 0.4671,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.519100546836853,
+      "rewards/margins": 0.7276883721351624,
+      "rewards/rejected": -0.20858784019947052,
+      "step": 101
+    },
+    {
+      "epoch": 0.9659090909090909,
+      "grad_norm": 9.94558673339714,
+      "learning_rate": 5.714285714285714e-08,
+      "logits/chosen": -1.5683315992355347,
+      "logits/rejected": -1.5737329721450806,
+      "logps/chosen": -380.98223876953125,
+      "logps/rejected": -148.6276092529297,
+      "loss": 0.4173,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6376638412475586,
+      "rewards/margins": 0.9165285229682922,
+      "rewards/rejected": -0.2788645625114441,
+      "step": 102
+    },
+    {
+      "epoch": 0.9753787878787878,
+      "grad_norm": 9.63450569739456,
+      "learning_rate": 5.6613756613756614e-08,
+      "logits/chosen": -1.5396722555160522,
+      "logits/rejected": -1.5394847393035889,
+      "logps/chosen": -313.8475341796875,
+      "logps/rejected": -103.21199035644531,
+      "loss": 0.4177,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.6744638681411743,
+      "rewards/margins": 0.8535541296005249,
+      "rewards/rejected": -0.17909026145935059,
+      "step": 103
+    },
+    {
+      "epoch": 0.9848484848484849,
+      "grad_norm": 10.940866953801127,
+      "learning_rate": 5.608465608465608e-08,
+      "logits/chosen": -1.5241594314575195,
+      "logits/rejected": -1.637131690979004,
+      "logps/chosen": -340.30072021484375,
+      "logps/rejected": -126.37461853027344,
+      "loss": 0.4387,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5279501080513,
+      "rewards/margins": 0.7279192805290222,
+      "rewards/rejected": -0.1999691277742386,
+      "step": 104
+    },
+    {
+      "epoch": 0.9943181818181818,
+      "grad_norm": 9.396057853660293,
+      "learning_rate": 5.5555555555555555e-08,
+      "logits/chosen": -1.4265496730804443,
+      "logits/rejected": -1.5132142305374146,
+      "logps/chosen": -269.0768127441406,
+      "logps/rejected": -128.25860595703125,
+      "loss": 0.4426,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.511640727519989,
+      "rewards/margins": 0.7144440412521362,
+      "rewards/rejected": -0.20280325412750244,
+      "step": 105
+    },
+    {
+      "epoch": 1.003787878787879,
+      "grad_norm": 7.831685341832891,
+      "learning_rate": 5.502645502645502e-08,
+      "logits/chosen": -1.5368983745574951,
+      "logits/rejected": -1.4743095636367798,
+      "logps/chosen": -280.42279052734375,
+      "logps/rejected": -121.66993713378906,
+      "loss": 0.4581,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.5917797684669495,
+      "rewards/margins": 0.7320090532302856,
+      "rewards/rejected": -0.14022931456565857,
+      "step": 106
+    },
+    {
+      "epoch": 1.0132575757575757,
+      "grad_norm": 8.940391987752884,
+      "learning_rate": 5.4497354497354495e-08,
+      "logits/chosen": -1.491202712059021,
+      "logits/rejected": -1.5914320945739746,
+      "logps/chosen": -321.1029052734375,
+      "logps/rejected": -147.7200164794922,
+      "loss": 0.4276,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.5120298862457275,
+      "rewards/margins": 0.6171775460243225,
+      "rewards/rejected": -0.10514764487743378,
+      "step": 107
+    },
+    {
+      "epoch": 1.0227272727272727,
+      "grad_norm": 7.697241759390277,
+      "learning_rate": 5.3968253968253965e-08,
+      "logits/chosen": -1.5066335201263428,
+      "logits/rejected": -1.528194546699524,
+      "logps/chosen": -292.2257385253906,
+      "logps/rejected": -174.4923553466797,
+      "loss": 0.4566,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5518472790718079,
+      "rewards/margins": 0.8220493197441101,
+      "rewards/rejected": -0.270202100276947,
+      "step": 108
+    },
+    {
+      "epoch": 1.0321969696969697,
+      "grad_norm": 7.9963617328841226,
+      "learning_rate": 5.343915343915344e-08,
+      "logits/chosen": -1.5413943529129028,
+      "logits/rejected": -1.6188589334487915,
+      "logps/chosen": -312.98065185546875,
+      "logps/rejected": -115.0594253540039,
+      "loss": 0.4259,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.5731326341629028,
+      "rewards/margins": 0.9009225964546204,
+      "rewards/rejected": -0.32778996229171753,
+      "step": 109
+    },
+    {
+      "epoch": 1.0416666666666667,
+      "grad_norm": 9.995827902761533,
+      "learning_rate": 5.2910052910052905e-08,
+      "logits/chosen": -1.6093279123306274,
+      "logits/rejected": -1.713158369064331,
+      "logps/chosen": -373.02288818359375,
+      "logps/rejected": -202.49237060546875,
+      "loss": 0.4242,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4924096167087555,
+      "rewards/margins": 0.7550744414329529,
+      "rewards/rejected": -0.2626648545265198,
+      "step": 110
+    },
+    {
+      "epoch": 1.0511363636363635,
+      "grad_norm": 6.7346694181058,
+      "learning_rate": 5.238095238095238e-08,
+      "logits/chosen": -1.571112871170044,
+      "logits/rejected": -1.5796878337860107,
+      "logps/chosen": -312.8564453125,
+      "logps/rejected": -136.47171020507812,
+      "loss": 0.4814,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5543137788772583,
+      "rewards/margins": 0.8626272082328796,
+      "rewards/rejected": -0.30831339955329895,
+      "step": 111
+    },
+    {
+      "epoch": 1.0606060606060606,
+      "grad_norm": 6.9001053561660015,
+      "learning_rate": 5.1851851851851846e-08,
+      "logits/chosen": -1.5522701740264893,
+      "logits/rejected": -1.6721560955047607,
+      "logps/chosen": -323.85064697265625,
+      "logps/rejected": -118.36842346191406,
+      "loss": 0.4539,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6884499192237854,
+      "rewards/margins": 0.9783046841621399,
+      "rewards/rejected": -0.28985467553138733,
+      "step": 112
+    },
+    {
+      "epoch": 1.0700757575757576,
+      "grad_norm": 8.261082045277393,
+      "learning_rate": 5.132275132275132e-08,
+      "logits/chosen": -1.5419814586639404,
+      "logits/rejected": -1.6027164459228516,
+      "logps/chosen": -262.14154052734375,
+      "logps/rejected": -85.59474182128906,
+      "loss": 0.42,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.4974077641963959,
+      "rewards/margins": 0.7110608220100403,
+      "rewards/rejected": -0.21365304291248322,
+      "step": 113
+    },
+    {
+      "epoch": 1.0795454545454546,
+      "grad_norm": 7.731297247151736,
+      "learning_rate": 5.0793650793650786e-08,
+      "logits/chosen": -1.5612366199493408,
+      "logits/rejected": -1.6464793682098389,
+      "logps/chosen": -212.521484375,
+      "logps/rejected": -63.86298751831055,
+      "loss": 0.408,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.43088382482528687,
+      "rewards/margins": 0.6654319167137146,
+      "rewards/rejected": -0.23454804718494415,
+      "step": 114
+    },
+    {
+      "epoch": 1.0890151515151516,
+      "grad_norm": 8.268687229033572,
+      "learning_rate": 5.026455026455026e-08,
+      "logits/chosen": -1.4982279539108276,
+      "logits/rejected": -1.5205714702606201,
+      "logps/chosen": -284.63787841796875,
+      "logps/rejected": -165.73988342285156,
+      "loss": 0.4353,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5018287301063538,
+      "rewards/margins": 0.7900735139846802,
+      "rewards/rejected": -0.2882448136806488,
+      "step": 115
+    },
+    {
+      "epoch": 1.0984848484848484,
+      "grad_norm": 7.675182494137689,
+      "learning_rate": 4.973544973544973e-08,
+      "logits/chosen": -1.3966763019561768,
+      "logits/rejected": -1.5771713256835938,
+      "logps/chosen": -306.68499755859375,
+      "logps/rejected": -174.55003356933594,
+      "loss": 0.3851,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.6541851162910461,
+      "rewards/margins": 0.9976884722709656,
+      "rewards/rejected": -0.34350335597991943,
+      "step": 116
+    },
+    {
+      "epoch": 1.1079545454545454,
+      "grad_norm": 7.288879950855222,
+      "learning_rate": 4.92063492063492e-08,
+      "logits/chosen": -1.6342532634735107,
+      "logits/rejected": -1.7281395196914673,
+      "logps/chosen": -312.36895751953125,
+      "logps/rejected": -198.22055053710938,
+      "loss": 0.4289,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6851889491081238,
+      "rewards/margins": 0.873565673828125,
+      "rewards/rejected": -0.18837669491767883,
+      "step": 117
+    },
+    {
+      "epoch": 1.1174242424242424,
+      "grad_norm": 7.471188637782996,
+      "learning_rate": 4.867724867724867e-08,
+      "logits/chosen": -1.5435848236083984,
+      "logits/rejected": -1.5624885559082031,
+      "logps/chosen": -276.4817810058594,
+      "logps/rejected": -140.15501403808594,
+      "loss": 0.4277,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.4910324513912201,
+      "rewards/margins": 0.6622015237808228,
+      "rewards/rejected": -0.17116901278495789,
+      "step": 118
+    },
+    {
+      "epoch": 1.1268939393939394,
+      "grad_norm": 6.073468496072038,
+      "learning_rate": 4.814814814814814e-08,
+      "logits/chosen": -1.549534559249878,
+      "logits/rejected": -1.7240833044052124,
+      "logps/chosen": -237.54714965820312,
+      "logps/rejected": -110.4820327758789,
+      "loss": 0.4246,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.43428367376327515,
+      "rewards/margins": 0.675010085105896,
+      "rewards/rejected": -0.2407263219356537,
+      "step": 119
+    },
+    {
+      "epoch": 1.1363636363636362,
+      "grad_norm": 6.229810704784674,
+      "learning_rate": 4.7619047619047613e-08,
+      "logits/chosen": -1.5288211107254028,
+      "logits/rejected": -1.6095609664916992,
+      "logps/chosen": -300.1777038574219,
+      "logps/rejected": -178.78512573242188,
+      "loss": 0.4332,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.45809292793273926,
+      "rewards/margins": 0.775213360786438,
+      "rewards/rejected": -0.31712037324905396,
+      "step": 120
+    },
+    {
+      "epoch": 1.1458333333333333,
+      "grad_norm": 6.153778206401315,
+      "learning_rate": 4.7089947089947084e-08,
+      "logits/chosen": -1.446118950843811,
+      "logits/rejected": -1.6403175592422485,
+      "logps/chosen": -261.62823486328125,
+      "logps/rejected": -110.68792724609375,
+      "loss": 0.4185,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5491897463798523,
+      "rewards/margins": 0.9256091117858887,
+      "rewards/rejected": -0.3764193654060364,
+      "step": 121
+    },
+    {
+      "epoch": 1.1553030303030303,
+      "grad_norm": 8.460733071100409,
+      "learning_rate": 4.6560846560846554e-08,
+      "logits/chosen": -1.5496113300323486,
+      "logits/rejected": -1.6899559497833252,
+      "logps/chosen": -339.3661193847656,
+      "logps/rejected": -121.66202545166016,
+      "loss": 0.3873,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.7354398369789124,
+      "rewards/margins": 1.1846985816955566,
+      "rewards/rejected": -0.44925880432128906,
+      "step": 122
+    },
+    {
+      "epoch": 1.1647727272727273,
+      "grad_norm": 7.315349675684202,
+      "learning_rate": 4.6031746031746024e-08,
+      "logits/chosen": -1.6141818761825562,
+      "logits/rejected": -1.6296923160552979,
+      "logps/chosen": -238.61257934570312,
+      "logps/rejected": -122.18446350097656,
+      "loss": 0.3669,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.37908780574798584,
+      "rewards/margins": 0.6342587471008301,
+      "rewards/rejected": -0.2551709711551666,
+      "step": 123
+    },
+    {
+      "epoch": 1.1742424242424243,
+      "grad_norm": 7.280988098748099,
+      "learning_rate": 4.55026455026455e-08,
+      "logits/chosen": -1.5299112796783447,
+      "logits/rejected": -1.5701675415039062,
+      "logps/chosen": -316.45538330078125,
+      "logps/rejected": -162.38204956054688,
+      "loss": 0.3653,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.628962516784668,
+      "rewards/margins": 1.195112943649292,
+      "rewards/rejected": -0.566150426864624,
+      "step": 124
+    },
+    {
+      "epoch": 1.183712121212121,
+      "grad_norm": 6.336179594301882,
+      "learning_rate": 4.497354497354497e-08,
+      "logits/chosen": -1.557908058166504,
+      "logits/rejected": -1.6704943180084229,
+      "logps/chosen": -325.92266845703125,
+      "logps/rejected": -167.68130493164062,
+      "loss": 0.3747,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.6076933741569519,
+      "rewards/margins": 0.9305548667907715,
+      "rewards/rejected": -0.3228614330291748,
+      "step": 125
+    },
+    {
+      "epoch": 1.1931818181818181,
+      "grad_norm": 6.805130421455224,
+      "learning_rate": 4.444444444444444e-08,
+      "logits/chosen": -1.4893940687179565,
+      "logits/rejected": -1.5416975021362305,
+      "logps/chosen": -164.71548461914062,
+      "logps/rejected": -102.4400634765625,
+      "loss": 0.3971,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2525160014629364,
+      "rewards/margins": 0.4948664605617523,
+      "rewards/rejected": -0.2423505038022995,
+      "step": 126
+    },
+    {
+      "epoch": 1.2026515151515151,
+      "grad_norm": 6.005160247149641,
+      "learning_rate": 4.391534391534391e-08,
+      "logits/chosen": -1.6034702062606812,
+      "logits/rejected": -1.7156803607940674,
+      "logps/chosen": -327.7132568359375,
+      "logps/rejected": -122.96253967285156,
+      "loss": 0.4048,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.611197292804718,
+      "rewards/margins": 0.990843653678894,
+      "rewards/rejected": -0.3796464204788208,
+      "step": 127
+    },
+    {
+      "epoch": 1.2121212121212122,
+      "grad_norm": 6.064066327086945,
+      "learning_rate": 4.338624338624338e-08,
+      "logits/chosen": -1.4845657348632812,
+      "logits/rejected": -1.6025197505950928,
+      "logps/chosen": -266.589599609375,
+      "logps/rejected": -185.93374633789062,
+      "loss": 0.3821,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.46795588731765747,
+      "rewards/margins": 0.7385295033454895,
+      "rewards/rejected": -0.27057361602783203,
+      "step": 128
+    },
+    {
+      "epoch": 1.2215909090909092,
+      "grad_norm": 7.118625992094876,
+      "learning_rate": 4.285714285714285e-08,
+      "logits/chosen": -1.5417770147323608,
+      "logits/rejected": -1.6401869058609009,
+      "logps/chosen": -364.9320983886719,
+      "logps/rejected": -157.976806640625,
+      "loss": 0.3608,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7918987274169922,
+      "rewards/margins": 1.300087571144104,
+      "rewards/rejected": -0.5081888437271118,
+      "step": 129
+    },
+    {
+      "epoch": 1.231060606060606,
+      "grad_norm": 5.890825166861846,
+      "learning_rate": 4.232804232804232e-08,
+      "logits/chosen": -1.6265586614608765,
+      "logits/rejected": -1.7117183208465576,
+      "logps/chosen": -360.13519287109375,
+      "logps/rejected": -177.2240753173828,
+      "loss": 0.4508,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.7431641221046448,
+      "rewards/margins": 1.2213959693908691,
+      "rewards/rejected": -0.47823190689086914,
+      "step": 130
+    },
+    {
+      "epoch": 1.240530303030303,
+      "grad_norm": 6.0694144195552315,
+      "learning_rate": 4.179894179894179e-08,
+      "logits/chosen": -1.5653178691864014,
+      "logits/rejected": -1.6441690921783447,
+      "logps/chosen": -263.4643859863281,
+      "logps/rejected": -122.57698059082031,
+      "loss": 0.4199,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.47927695512771606,
+      "rewards/margins": 0.9024354219436646,
+      "rewards/rejected": -0.4231584668159485,
+      "step": 131
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 5.640397757089755,
+      "learning_rate": 4.126984126984127e-08,
+      "logits/chosen": -1.5248239040374756,
+      "logits/rejected": -1.5807774066925049,
+      "logps/chosen": -232.2427520751953,
+      "logps/rejected": -182.7039794921875,
+      "loss": 0.4312,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.33463865518569946,
+      "rewards/margins": 0.5401297807693481,
+      "rewards/rejected": -0.20549115538597107,
+      "step": 132
+    },
+    {
+      "epoch": 1.259469696969697,
+      "grad_norm": 6.188835951236042,
+      "learning_rate": 4.074074074074074e-08,
+      "logits/chosen": -1.5644571781158447,
+      "logits/rejected": -1.5982444286346436,
+      "logps/chosen": -316.8279724121094,
+      "logps/rejected": -114.7803726196289,
+      "loss": 0.3815,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6954594254493713,
+      "rewards/margins": 1.1970874071121216,
+      "rewards/rejected": -0.5016279816627502,
+      "step": 133
+    },
+    {
+      "epoch": 1.268939393939394,
+      "grad_norm": 5.913640581542438,
+      "learning_rate": 4.021164021164021e-08,
+      "logits/chosen": -1.5394070148468018,
+      "logits/rejected": -1.6787744760513306,
+      "logps/chosen": -210.8607940673828,
+      "logps/rejected": -104.78348541259766,
+      "loss": 0.4341,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.422776460647583,
+      "rewards/margins": 0.6558708548545837,
+      "rewards/rejected": -0.2330944538116455,
+      "step": 134
+    },
+    {
+      "epoch": 1.2784090909090908,
+      "grad_norm": 5.979621420732666,
+      "learning_rate": 3.968253968253968e-08,
+      "logits/chosen": -1.5477142333984375,
+      "logits/rejected": -1.659967064857483,
+      "logps/chosen": -297.19622802734375,
+      "logps/rejected": -153.1532745361328,
+      "loss": 0.3554,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6835674047470093,
+      "rewards/margins": 1.379019021987915,
+      "rewards/rejected": -0.695451557636261,
+      "step": 135
+    },
+    {
+      "epoch": 1.2878787878787878,
+      "grad_norm": 5.932788102313539,
+      "learning_rate": 3.915343915343915e-08,
+      "logits/chosen": -1.4924932718276978,
+      "logits/rejected": -1.6393858194351196,
+      "logps/chosen": -246.09750366210938,
+      "logps/rejected": -120.7838363647461,
+      "loss": 0.3555,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5313190221786499,
+      "rewards/margins": 0.9934673309326172,
+      "rewards/rejected": -0.4621482789516449,
+      "step": 136
+    },
+    {
+      "epoch": 1.2973484848484849,
+      "grad_norm": 14.930220486136447,
+      "learning_rate": 3.862433862433862e-08,
+      "logits/chosen": -1.5214557647705078,
+      "logits/rejected": -1.6087700128555298,
+      "logps/chosen": -384.80938720703125,
+      "logps/rejected": -210.44235229492188,
+      "loss": 0.3889,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.7282799482345581,
+      "rewards/margins": 1.31002676486969,
+      "rewards/rejected": -0.5817468762397766,
+      "step": 137
+    },
+    {
+      "epoch": 1.3068181818181819,
+      "grad_norm": 5.971012894528325,
+      "learning_rate": 3.809523809523809e-08,
+      "logits/chosen": -1.5881704092025757,
+      "logits/rejected": -1.569000005722046,
+      "logps/chosen": -297.71490478515625,
+      "logps/rejected": -158.21453857421875,
+      "loss": 0.3827,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.487607479095459,
+      "rewards/margins": 0.8952295184135437,
+      "rewards/rejected": -0.40762215852737427,
+      "step": 138
+    },
+    {
+      "epoch": 1.316287878787879,
+      "grad_norm": 5.470550316559649,
+      "learning_rate": 3.756613756613756e-08,
+      "logits/chosen": -1.5592622756958008,
+      "logits/rejected": -1.6217191219329834,
+      "logps/chosen": -208.8889923095703,
+      "logps/rejected": -99.66307067871094,
+      "loss": 0.3742,
+      "rewards/accuracies": 0.9333332777023315,
+      "rewards/chosen": 0.47726184129714966,
+      "rewards/margins": 0.8086546063423157,
+      "rewards/rejected": -0.3313927948474884,
+      "step": 139
+    },
+    {
+      "epoch": 1.3257575757575757,
+      "grad_norm": 5.53149417210968,
+      "learning_rate": 3.7037037037037036e-08,
+      "logits/chosen": -1.5759552717208862,
+      "logits/rejected": -1.6656872034072876,
+      "logps/chosen": -282.05584716796875,
+      "logps/rejected": -137.61387634277344,
+      "loss": 0.3673,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5351212024688721,
+      "rewards/margins": 0.9544070363044739,
+      "rewards/rejected": -0.41928577423095703,
+      "step": 140
+    },
+    {
+      "epoch": 1.3352272727272727,
+      "grad_norm": 5.589581006217635,
+      "learning_rate": 3.6507936507936506e-08,
+      "logits/chosen": -1.6098802089691162,
+      "logits/rejected": -1.6987054347991943,
+      "logps/chosen": -248.8202362060547,
+      "logps/rejected": -105.59635925292969,
+      "loss": 0.4226,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5064533352851868,
+      "rewards/margins": 0.982274055480957,
+      "rewards/rejected": -0.47582077980041504,
+      "step": 141
+    },
+    {
+      "epoch": 1.3446969696969697,
+      "grad_norm": 6.277665119781322,
+      "learning_rate": 3.5978835978835977e-08,
+      "logits/chosen": -1.5494630336761475,
+      "logits/rejected": -1.6693788766860962,
+      "logps/chosen": -351.03125,
+      "logps/rejected": -129.0759735107422,
+      "loss": 0.3673,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7771791219711304,
+      "rewards/margins": 1.4887819290161133,
+      "rewards/rejected": -0.7116026878356934,
+      "step": 142
+    },
+    {
+      "epoch": 1.3541666666666667,
+      "grad_norm": 5.8355671091327075,
+      "learning_rate": 3.544973544973545e-08,
+      "logits/chosen": -1.559160828590393,
+      "logits/rejected": -1.5491154193878174,
+      "logps/chosen": -335.740234375,
+      "logps/rejected": -170.25552368164062,
+      "loss": 0.4013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7102228999137878,
+      "rewards/margins": 1.2653253078460693,
+      "rewards/rejected": -0.555102527141571,
+      "step": 143
+    },
+    {
+      "epoch": 1.3636363636363638,
+      "grad_norm": 5.768525584457291,
+      "learning_rate": 3.492063492063492e-08,
+      "logits/chosen": -1.6074180603027344,
+      "logits/rejected": -1.7357871532440186,
+      "logps/chosen": -264.52679443359375,
+      "logps/rejected": -100.31982421875,
+      "loss": 0.408,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.4864141345024109,
+      "rewards/margins": 0.7962800860404968,
+      "rewards/rejected": -0.30986595153808594,
+      "step": 144
+    },
+    {
+      "epoch": 1.3731060606060606,
+      "grad_norm": 5.592283665576824,
+      "learning_rate": 3.439153439153439e-08,
+      "logits/chosen": -1.5794652700424194,
+      "logits/rejected": -1.6425234079360962,
+      "logps/chosen": -305.49005126953125,
+      "logps/rejected": -128.72915649414062,
+      "loss": 0.3636,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.6161352396011353,
+      "rewards/margins": 1.2325931787490845,
+      "rewards/rejected": -0.6164579391479492,
+      "step": 145
+    },
+    {
+      "epoch": 1.3825757575757576,
+      "grad_norm": 5.2043920846765515,
+      "learning_rate": 3.386243386243386e-08,
+      "logits/chosen": -1.6179447174072266,
+      "logits/rejected": -1.7832599878311157,
+      "logps/chosen": -317.20703125,
+      "logps/rejected": -164.9311065673828,
+      "loss": 0.3351,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.6158957481384277,
+      "rewards/margins": 1.1830036640167236,
+      "rewards/rejected": -0.5671079158782959,
+      "step": 146
+    },
+    {
+      "epoch": 1.3920454545454546,
+      "grad_norm": 5.389376859152257,
+      "learning_rate": 3.333333333333333e-08,
+      "logits/chosen": -1.5436028242111206,
+      "logits/rejected": -1.7525603771209717,
+      "logps/chosen": -217.86569213867188,
+      "logps/rejected": -93.09127044677734,
+      "loss": 0.3912,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.4181972146034241,
+      "rewards/margins": 0.8372113108634949,
+      "rewards/rejected": -0.4190141260623932,
+      "step": 147
+    },
+    {
+      "epoch": 1.4015151515151514,
+      "grad_norm": 5.429011665107999,
+      "learning_rate": 3.2804232804232804e-08,
+      "logits/chosen": -1.5702577829360962,
+      "logits/rejected": -1.6486690044403076,
+      "logps/chosen": -319.3020324707031,
+      "logps/rejected": -182.95730590820312,
+      "loss": 0.3457,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.5197921395301819,
+      "rewards/margins": 0.8637401461601257,
+      "rewards/rejected": -0.34394803643226624,
+      "step": 148
+    },
+    {
+      "epoch": 1.4109848484848486,
+      "grad_norm": 5.368022221479501,
+      "learning_rate": 3.2275132275132274e-08,
+      "logits/chosen": -1.4937713146209717,
+      "logits/rejected": -1.5651482343673706,
+      "logps/chosen": -234.36328125,
+      "logps/rejected": -148.08285522460938,
+      "loss": 0.3444,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.23662462830543518,
+      "rewards/margins": 0.6261420845985413,
+      "rewards/rejected": -0.3895173966884613,
+      "step": 149
+    },
+    {
+      "epoch": 1.4204545454545454,
+      "grad_norm": 5.493890942440773,
+      "learning_rate": 3.1746031746031744e-08,
+      "logits/chosen": -1.6106767654418945,
+      "logits/rejected": -1.5224909782409668,
+      "logps/chosen": -302.6184997558594,
+      "logps/rejected": -146.08517456054688,
+      "loss": 0.3663,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48458927869796753,
+      "rewards/margins": 0.9824308156967163,
+      "rewards/rejected": -0.49784159660339355,
+      "step": 150
+    },
+    {
+      "epoch": 1.4299242424242424,
+      "grad_norm": 5.480765334687315,
+      "learning_rate": 3.1216931216931215e-08,
+      "logits/chosen": -1.5799682140350342,
+      "logits/rejected": -1.7092950344085693,
+      "logps/chosen": -245.9119110107422,
+      "logps/rejected": -150.98989868164062,
+      "loss": 0.4027,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4121614396572113,
+      "rewards/margins": 0.8830633163452148,
+      "rewards/rejected": -0.47090187668800354,
+      "step": 151
+    },
+    {
+      "epoch": 1.4393939393939394,
+      "grad_norm": 6.044424074410309,
+      "learning_rate": 3.0687830687830685e-08,
+      "logits/chosen": -1.5871868133544922,
+      "logits/rejected": -1.6946001052856445,
+      "logps/chosen": -346.11822509765625,
+      "logps/rejected": -244.7041778564453,
+      "loss": 0.4391,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.3353393077850342,
+      "rewards/margins": 0.7316913604736328,
+      "rewards/rejected": -0.39635199308395386,
+      "step": 152
+    },
+    {
+      "epoch": 1.4488636363636362,
+      "grad_norm": 4.974140552150539,
+      "learning_rate": 3.0158730158730155e-08,
+      "logits/chosen": -1.5874955654144287,
+      "logits/rejected": -1.64119553565979,
+      "logps/chosen": -270.36883544921875,
+      "logps/rejected": -129.67535400390625,
+      "loss": 0.3404,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7120252251625061,
+      "rewards/margins": 1.3487727642059326,
+      "rewards/rejected": -0.6367474794387817,
+      "step": 153
+    },
+    {
+      "epoch": 1.4583333333333333,
+      "grad_norm": 5.350141771926167,
+      "learning_rate": 2.9629629629629625e-08,
+      "logits/chosen": -1.5650367736816406,
+      "logits/rejected": -1.6837724447250366,
+      "logps/chosen": -271.3443298339844,
+      "logps/rejected": -136.472412109375,
+      "loss": 0.3361,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.6383367776870728,
+      "rewards/margins": 1.251798391342163,
+      "rewards/rejected": -0.6134616136550903,
+      "step": 154
+    },
+    {
+      "epoch": 1.4678030303030303,
+      "grad_norm": 5.429121594474915,
+      "learning_rate": 2.91005291005291e-08,
+      "logits/chosen": -1.568342924118042,
+      "logits/rejected": -1.6543916463851929,
+      "logps/chosen": -284.5910339355469,
+      "logps/rejected": -152.6836395263672,
+      "loss": 0.3293,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5576558113098145,
+      "rewards/margins": 0.9742603302001953,
+      "rewards/rejected": -0.41660451889038086,
+      "step": 155
+    },
+    {
+      "epoch": 1.4772727272727273,
+      "grad_norm": 5.748560761291252,
+      "learning_rate": 2.857142857142857e-08,
+      "logits/chosen": -1.4832626581192017,
+      "logits/rejected": -1.566612958908081,
+      "logps/chosen": -323.40142822265625,
+      "logps/rejected": -178.55923461914062,
+      "loss": 0.421,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.6060717105865479,
+      "rewards/margins": 1.2732772827148438,
+      "rewards/rejected": -0.6672054529190063,
+      "step": 156
+    },
+    {
+      "epoch": 1.4867424242424243,
+      "grad_norm": 5.310060511796978,
+      "learning_rate": 2.804232804232804e-08,
+      "logits/chosen": -1.5736675262451172,
+      "logits/rejected": -1.6186530590057373,
+      "logps/chosen": -316.97698974609375,
+      "logps/rejected": -155.43638610839844,
+      "loss": 0.3521,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6889209747314453,
+      "rewards/margins": 1.1352983713150024,
+      "rewards/rejected": -0.44637736678123474,
+      "step": 157
+    },
+    {
+      "epoch": 1.496212121212121,
+      "grad_norm": 5.118308720530541,
+      "learning_rate": 2.751322751322751e-08,
+      "logits/chosen": -1.560807704925537,
+      "logits/rejected": -1.7643381357192993,
+      "logps/chosen": -402.16632080078125,
+      "logps/rejected": -132.56011962890625,
+      "loss": 0.3576,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7692705392837524,
+      "rewards/margins": 1.3775382041931152,
+      "rewards/rejected": -0.6082676649093628,
+      "step": 158
+    },
+    {
+      "epoch": 1.5056818181818183,
+      "grad_norm": 5.9001461393206505,
+      "learning_rate": 2.6984126984126982e-08,
+      "logits/chosen": -1.5756150484085083,
+      "logits/rejected": -1.6618843078613281,
+      "logps/chosen": -144.79986572265625,
+      "logps/rejected": -142.03944396972656,
+      "loss": 0.3529,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.27606311440467834,
+      "rewards/margins": 0.5494292378425598,
+      "rewards/rejected": -0.27336612343788147,
+      "step": 159
+    },
+    {
+      "epoch": 1.5151515151515151,
+      "grad_norm": 5.919050512179857,
+      "learning_rate": 2.6455026455026453e-08,
+      "logits/chosen": -1.596430778503418,
+      "logits/rejected": -1.7081763744354248,
+      "logps/chosen": -381.9976806640625,
+      "logps/rejected": -168.22308349609375,
+      "loss": 0.3632,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.7063287496566772,
+      "rewards/margins": 1.4756296873092651,
+      "rewards/rejected": -0.7693011164665222,
+      "step": 160
+    },
+    {
+      "epoch": 1.5246212121212122,
+      "grad_norm": 6.771452793232538,
+      "learning_rate": 2.5925925925925923e-08,
+      "logits/chosen": -1.6167709827423096,
+      "logits/rejected": -1.702599287033081,
+      "logps/chosen": -272.24017333984375,
+      "logps/rejected": -205.1815185546875,
+      "loss": 0.3699,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.6321600675582886,
+      "rewards/margins": 1.0702993869781494,
+      "rewards/rejected": -0.43813928961753845,
+      "step": 161
+    },
+    {
+      "epoch": 1.5340909090909092,
+      "grad_norm": 5.844973840956046,
+      "learning_rate": 2.5396825396825393e-08,
+      "logits/chosen": -1.5797332525253296,
+      "logits/rejected": -1.6782172918319702,
+      "logps/chosen": -294.10540771484375,
+      "logps/rejected": -178.05677795410156,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": 0.4381368160247803,
+      "rewards/margins": 0.7244521975517273,
+      "rewards/rejected": -0.28631535172462463,
+      "step": 162
+    },
+    {
+      "epoch": 1.543560606060606,
+      "grad_norm": 5.9799282018427355,
+      "learning_rate": 2.4867724867724866e-08,
+      "logits/chosen": -1.5741981267929077,
+      "logits/rejected": -1.7271267175674438,
+      "logps/chosen": -279.97894287109375,
+      "logps/rejected": -155.4124298095703,
+      "loss": 0.3788,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5363057255744934,
+      "rewards/margins": 1.2445316314697266,
+      "rewards/rejected": -0.7082260251045227,
+      "step": 163
+    },
+    {
+      "epoch": 1.553030303030303,
+      "grad_norm": 5.688137477051427,
+      "learning_rate": 2.4338624338624337e-08,
+      "logits/chosen": -1.5423920154571533,
+      "logits/rejected": -1.7703033685684204,
+      "logps/chosen": -318.68963623046875,
+      "logps/rejected": -139.80978393554688,
+      "loss": 0.3199,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7159373760223389,
+      "rewards/margins": 1.3830876350402832,
+      "rewards/rejected": -0.6671503186225891,
+      "step": 164
+    },
+    {
+      "epoch": 1.5625,
+      "grad_norm": 5.159199147076709,
+      "learning_rate": 2.3809523809523807e-08,
+      "logits/chosen": -1.6055580377578735,
+      "logits/rejected": -1.7235870361328125,
+      "logps/chosen": -283.23931884765625,
+      "logps/rejected": -170.2466278076172,
+      "loss": 0.3879,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5190203785896301,
+      "rewards/margins": 0.8587003946304321,
+      "rewards/rejected": -0.3396799862384796,
+      "step": 165
+    },
+    {
+      "epoch": 1.571969696969697,
+      "grad_norm": 5.51593434736744,
+      "learning_rate": 2.3280423280423277e-08,
+      "logits/chosen": -1.6276752948760986,
+      "logits/rejected": -1.705373764038086,
+      "logps/chosen": -361.263671875,
+      "logps/rejected": -157.3756866455078,
+      "loss": 0.3503,
+      "rewards/accuracies": 0.9333332777023315,
+      "rewards/chosen": 0.813951313495636,
+      "rewards/margins": 1.6092281341552734,
+      "rewards/rejected": -0.7952768206596375,
+      "step": 166
+    },
+    {
+      "epoch": 1.581439393939394,
+      "grad_norm": 5.105458599961924,
+      "learning_rate": 2.275132275132275e-08,
+      "logits/chosen": -1.4909486770629883,
+      "logits/rejected": -1.6180715560913086,
+      "logps/chosen": -216.1019287109375,
+      "logps/rejected": -78.71076965332031,
+      "loss": 0.3386,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.488716185092926,
+      "rewards/margins": 1.0591917037963867,
+      "rewards/rejected": -0.5704755187034607,
+      "step": 167
+    },
+    {
+      "epoch": 1.5909090909090908,
+      "grad_norm": 4.489310539882785,
+      "learning_rate": 2.222222222222222e-08,
+      "logits/chosen": -1.5696783065795898,
+      "logits/rejected": -1.6010246276855469,
+      "logps/chosen": -306.27294921875,
+      "logps/rejected": -123.8807373046875,
+      "loss": 0.3769,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7523918151855469,
+      "rewards/margins": 1.3328092098236084,
+      "rewards/rejected": -0.5804173946380615,
+      "step": 168
+    },
+    {
+      "epoch": 1.6003787878787878,
+      "grad_norm": 5.400154224053731,
+      "learning_rate": 2.169312169312169e-08,
+      "logits/chosen": -1.5251656770706177,
+      "logits/rejected": -1.6274874210357666,
+      "logps/chosen": -130.54373168945312,
+      "logps/rejected": -54.27812576293945,
+      "loss": 0.4268,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.27597469091415405,
+      "rewards/margins": 0.5285931825637817,
+      "rewards/rejected": -0.2526185214519501,
+      "step": 169
+    },
+    {
+      "epoch": 1.6098484848484849,
+      "grad_norm": 4.692184736415839,
+      "learning_rate": 2.116402116402116e-08,
+      "logits/chosen": -1.5292972326278687,
+      "logits/rejected": -1.6729373931884766,
+      "logps/chosen": -360.6134948730469,
+      "logps/rejected": -151.4227294921875,
+      "loss": 0.3845,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8804464340209961,
+      "rewards/margins": 1.6148141622543335,
+      "rewards/rejected": -0.7343679666519165,
+      "step": 170
+    },
+    {
+      "epoch": 1.6193181818181817,
+      "grad_norm": 5.938380627671962,
+      "learning_rate": 2.0634920634920634e-08,
+      "logits/chosen": -1.5955592393875122,
+      "logits/rejected": -1.7036798000335693,
+      "logps/chosen": -384.96380615234375,
+      "logps/rejected": -170.1694793701172,
+      "loss": 0.3205,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8083392977714539,
+      "rewards/margins": 1.5802361965179443,
+      "rewards/rejected": -0.7718968391418457,
+      "step": 171
+    },
+    {
+      "epoch": 1.628787878787879,
+      "grad_norm": 5.0095603748386415,
+      "learning_rate": 2.0105820105820104e-08,
+      "logits/chosen": -1.559670329093933,
+      "logits/rejected": -1.6990169286727905,
+      "logps/chosen": -260.5107421875,
+      "logps/rejected": -138.97021484375,
+      "loss": 0.3619,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6676410436630249,
+      "rewards/margins": 1.1652050018310547,
+      "rewards/rejected": -0.49756401777267456,
+      "step": 172
+    },
+    {
+      "epoch": 1.6382575757575757,
+      "grad_norm": 5.046726473189086,
+      "learning_rate": 1.9576719576719575e-08,
+      "logits/chosen": -1.5894399881362915,
+      "logits/rejected": -1.711214303970337,
+      "logps/chosen": -308.7052307128906,
+      "logps/rejected": -193.82850646972656,
+      "loss": 0.3758,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.514888346195221,
+      "rewards/margins": 1.0377848148345947,
+      "rewards/rejected": -0.5228964686393738,
+      "step": 173
+    },
+    {
+      "epoch": 1.6477272727272727,
+      "grad_norm": 10.53036659721855,
+      "learning_rate": 1.9047619047619045e-08,
+      "logits/chosen": -1.5739425420761108,
+      "logits/rejected": -1.5461231470108032,
+      "logps/chosen": -272.7663269042969,
+      "logps/rejected": -152.17575073242188,
+      "loss": 0.3848,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.457489013671875,
+      "rewards/margins": 1.1182512044906616,
+      "rewards/rejected": -0.6607621908187866,
+      "step": 174
+    },
+    {
+      "epoch": 1.6571969696969697,
+      "grad_norm": 5.78981346354209,
+      "learning_rate": 1.8518518518518518e-08,
+      "logits/chosen": -1.5584112405776978,
+      "logits/rejected": -1.66403329372406,
+      "logps/chosen": -247.25527954101562,
+      "logps/rejected": -88.81005096435547,
+      "loss": 0.3545,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.38863998651504517,
+      "rewards/margins": 0.8395411372184753,
+      "rewards/rejected": -0.45090118050575256,
+      "step": 175
+    },
+    {
+      "epoch": 1.6666666666666665,
+      "grad_norm": 5.544069471962977,
+      "learning_rate": 1.7989417989417988e-08,
+      "logits/chosen": -1.5310989618301392,
+      "logits/rejected": -1.6411716938018799,
+      "logps/chosen": -177.37261962890625,
+      "logps/rejected": -94.64937591552734,
+      "loss": 0.3993,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.40063905715942383,
+      "rewards/margins": 0.7527322769165039,
+      "rewards/rejected": -0.35209327936172485,
+      "step": 176
+    },
+    {
+      "epoch": 1.6761363636363638,
+      "grad_norm": 6.870515621075784,
+      "learning_rate": 1.746031746031746e-08,
+      "logits/chosen": -1.6042630672454834,
+      "logits/rejected": -1.7060787677764893,
+      "logps/chosen": -329.9851989746094,
+      "logps/rejected": -170.91259765625,
+      "loss": 0.3258,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6290345191955566,
+      "rewards/margins": 1.0913975238800049,
+      "rewards/rejected": -0.4623629152774811,
+      "step": 177
+    },
+    {
+      "epoch": 1.6856060606060606,
+      "grad_norm": 5.283044248226327,
+      "learning_rate": 1.693121693121693e-08,
+      "logits/chosen": -1.5727237462997437,
+      "logits/rejected": -1.670090675354004,
+      "logps/chosen": -242.1526336669922,
+      "logps/rejected": -123.24870300292969,
+      "loss": 0.3156,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5657342076301575,
+      "rewards/margins": 1.0815435647964478,
+      "rewards/rejected": -0.5158092379570007,
+      "step": 178
+    },
+    {
+      "epoch": 1.6950757575757576,
+      "grad_norm": 4.714665015837958,
+      "learning_rate": 1.6402116402116402e-08,
+      "logits/chosen": -1.6179364919662476,
+      "logits/rejected": -1.6783673763275146,
+      "logps/chosen": -268.26666259765625,
+      "logps/rejected": -125.2500991821289,
+      "loss": 0.3028,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6312516331672668,
+      "rewards/margins": 1.2757152318954468,
+      "rewards/rejected": -0.6444636583328247,
+      "step": 179
+    },
+    {
+      "epoch": 1.7045454545454546,
+      "grad_norm": 5.48800386545785,
+      "learning_rate": 1.5873015873015872e-08,
+      "logits/chosen": -1.587045431137085,
+      "logits/rejected": -1.5843976736068726,
+      "logps/chosen": -314.2275085449219,
+      "logps/rejected": -202.48590087890625,
+      "loss": 0.3407,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4960121214389801,
+      "rewards/margins": 1.1439507007598877,
+      "rewards/rejected": -0.64793860912323,
+      "step": 180
+    },
+    {
+      "epoch": 1.7140151515151514,
+      "grad_norm": 4.904790456188903,
+      "learning_rate": 1.5343915343915342e-08,
+      "logits/chosen": -1.5327612161636353,
+      "logits/rejected": -1.6153557300567627,
+      "logps/chosen": -337.2438659667969,
+      "logps/rejected": -185.306396484375,
+      "loss": 0.3592,
+      "rewards/accuracies": 0.9333332777023315,
+      "rewards/chosen": 0.5627692937850952,
+      "rewards/margins": 1.312888741493225,
+      "rewards/rejected": -0.7501195669174194,
+      "step": 181
+    },
+    {
+      "epoch": 1.7234848484848486,
+      "grad_norm": 4.656789025606906,
+      "learning_rate": 1.4814814814814813e-08,
+      "logits/chosen": -1.6096194982528687,
+      "logits/rejected": -1.7528839111328125,
+      "logps/chosen": -336.3609313964844,
+      "logps/rejected": -170.637939453125,
+      "loss": 0.3605,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5875322818756104,
+      "rewards/margins": 1.3081997632980347,
+      "rewards/rejected": -0.7206674218177795,
+      "step": 182
+    },
+    {
+      "epoch": 1.7329545454545454,
+      "grad_norm": 5.444360894971359,
+      "learning_rate": 1.4285714285714284e-08,
+      "logits/chosen": -1.5957181453704834,
+      "logits/rejected": -1.64398992061615,
+      "logps/chosen": -289.0057678222656,
+      "logps/rejected": -167.2021484375,
+      "loss": 0.3833,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.6649573445320129,
+      "rewards/margins": 1.3411014080047607,
+      "rewards/rejected": -0.6761440634727478,
+      "step": 183
+    },
+    {
+      "epoch": 1.7424242424242424,
+      "grad_norm": 4.598365388289265,
+      "learning_rate": 1.3756613756613755e-08,
+      "logits/chosen": -1.6330028772354126,
+      "logits/rejected": -1.6998701095581055,
+      "logps/chosen": -245.7683868408203,
+      "logps/rejected": -142.4471435546875,
+      "loss": 0.392,
+      "rewards/accuracies": 0.73333340883255,
+      "rewards/chosen": 0.36274439096450806,
+      "rewards/margins": 0.7017933130264282,
+      "rewards/rejected": -0.3390488922595978,
+      "step": 184
+    },
+    {
+      "epoch": 1.7518939393939394,
+      "grad_norm": 5.234791929984342,
+      "learning_rate": 1.3227513227513226e-08,
+      "logits/chosen": -1.5748271942138672,
+      "logits/rejected": -1.6001355648040771,
+      "logps/chosen": -276.45074462890625,
+      "logps/rejected": -114.4822006225586,
+      "loss": 0.3666,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6184806227684021,
+      "rewards/margins": 1.1440364122390747,
+      "rewards/rejected": -0.5255557894706726,
+      "step": 185
+    },
+    {
+      "epoch": 1.7613636363636362,
+      "grad_norm": 5.537848306393154,
+      "learning_rate": 1.2698412698412696e-08,
+      "logits/chosen": -1.5969727039337158,
+      "logits/rejected": -1.673282265663147,
+      "logps/chosen": -222.10464477539062,
+      "logps/rejected": -115.9425048828125,
+      "loss": 0.3751,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.4460936188697815,
+      "rewards/margins": 0.7085092663764954,
+      "rewards/rejected": -0.26241564750671387,
+      "step": 186
+    },
+    {
+      "epoch": 1.7708333333333335,
+      "grad_norm": 4.835457356224698,
+      "learning_rate": 1.2169312169312168e-08,
+      "logits/chosen": -1.5568116903305054,
+      "logits/rejected": -1.554927110671997,
+      "logps/chosen": -256.4602966308594,
+      "logps/rejected": -167.30160522460938,
+      "loss": 0.3714,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4909042418003082,
+      "rewards/margins": 1.2990154027938843,
+      "rewards/rejected": -0.8081111907958984,
+      "step": 187
+    },
+    {
+      "epoch": 1.7803030303030303,
+      "grad_norm": 5.606816215751534,
+      "learning_rate": 1.1640211640211638e-08,
+      "logits/chosen": -1.578005313873291,
+      "logits/rejected": -1.5507481098175049,
+      "logps/chosen": -302.0408630371094,
+      "logps/rejected": -188.82948303222656,
+      "loss": 0.3905,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.4071738123893738,
+      "rewards/margins": 0.8548933267593384,
+      "rewards/rejected": -0.44771942496299744,
+      "step": 188
+    },
+    {
+      "epoch": 1.7897727272727273,
+      "grad_norm": 5.104068105653885,
+      "learning_rate": 1.111111111111111e-08,
+      "logits/chosen": -1.5117594003677368,
+      "logits/rejected": -1.6626886129379272,
+      "logps/chosen": -316.0799255371094,
+      "logps/rejected": -152.69705200195312,
+      "loss": 0.3749,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.49852848052978516,
+      "rewards/margins": 0.9504121541976929,
+      "rewards/rejected": -0.45188361406326294,
+      "step": 189
+    },
+    {
+      "epoch": 1.7992424242424243,
+      "grad_norm": 4.472316490828338,
+      "learning_rate": 1.058201058201058e-08,
+      "logits/chosen": -1.62430739402771,
+      "logits/rejected": -1.6985547542572021,
+      "logps/chosen": -448.1697692871094,
+      "logps/rejected": -224.6002655029297,
+      "loss": 0.3446,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8359583616256714,
+      "rewards/margins": 1.992840051651001,
+      "rewards/rejected": -1.1568816900253296,
+      "step": 190
+    },
+    {
+      "epoch": 1.808712121212121,
+      "grad_norm": 5.810880253949101,
+      "learning_rate": 1.0052910052910052e-08,
+      "logits/chosen": -1.5918437242507935,
+      "logits/rejected": -1.765323281288147,
+      "logps/chosen": -375.71429443359375,
+      "logps/rejected": -159.8873291015625,
+      "loss": 0.399,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.9120176434516907,
+      "rewards/margins": 1.7272794246673584,
+      "rewards/rejected": -0.8152618408203125,
+      "step": 191
+    },
+    {
+      "epoch": 1.8181818181818183,
+      "grad_norm": 4.244845222552646,
+      "learning_rate": 9.523809523809522e-09,
+      "logits/chosen": -1.5941245555877686,
+      "logits/rejected": -1.6600911617279053,
+      "logps/chosen": -305.6556701660156,
+      "logps/rejected": -134.87252807617188,
+      "loss": 0.3493,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.5449376106262207,
+      "rewards/margins": 1.1537270545959473,
+      "rewards/rejected": -0.6087895631790161,
+      "step": 192
+    },
+    {
+      "epoch": 1.8276515151515151,
+      "grad_norm": 4.738897289851384,
+      "learning_rate": 8.994708994708994e-09,
+      "logits/chosen": -1.5437757968902588,
+      "logits/rejected": -1.6903842687606812,
+      "logps/chosen": -321.27752685546875,
+      "logps/rejected": -126.69343566894531,
+      "loss": 0.3567,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6183031797409058,
+      "rewards/margins": 1.4520056247711182,
+      "rewards/rejected": -0.8337022662162781,
+      "step": 193
+    },
+    {
+      "epoch": 1.8371212121212122,
+      "grad_norm": 5.490859875590591,
+      "learning_rate": 8.465608465608464e-09,
+      "logits/chosen": -1.6326143741607666,
+      "logits/rejected": -1.7613766193389893,
+      "logps/chosen": -266.3793029785156,
+      "logps/rejected": -130.99026489257812,
+      "loss": 0.4226,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5176450610160828,
+      "rewards/margins": 0.859043300151825,
+      "rewards/rejected": -0.3413982093334198,
+      "step": 194
+    },
+    {
+      "epoch": 1.8465909090909092,
+      "grad_norm": 5.583102046304065,
+      "learning_rate": 7.936507936507936e-09,
+      "logits/chosen": -1.5793282985687256,
+      "logits/rejected": -1.789536714553833,
+      "logps/chosen": -386.55694580078125,
+      "logps/rejected": -170.34373474121094,
+      "loss": 0.3427,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5163098573684692,
+      "rewards/margins": 0.9769344329833984,
+      "rewards/rejected": -0.4606245458126068,
+      "step": 195
+    },
+    {
+      "epoch": 1.856060606060606,
+      "grad_norm": 6.799122206347572,
+      "learning_rate": 7.407407407407406e-09,
+      "logits/chosen": -1.5655455589294434,
+      "logits/rejected": -1.7494919300079346,
+      "logps/chosen": -340.42059326171875,
+      "logps/rejected": -156.17343139648438,
+      "loss": 0.3914,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.6122487783432007,
+      "rewards/margins": 1.0115610361099243,
+      "rewards/rejected": -0.39931216835975647,
+      "step": 196
+    },
+    {
+      "epoch": 1.865530303030303,
+      "grad_norm": 6.33489638516624,
+      "learning_rate": 6.878306878306877e-09,
+      "logits/chosen": -1.6181929111480713,
+      "logits/rejected": -1.7337757349014282,
+      "logps/chosen": -297.11236572265625,
+      "logps/rejected": -133.5535430908203,
+      "loss": 0.3948,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.5498533844947815,
+      "rewards/margins": 1.0373098850250244,
+      "rewards/rejected": -0.48745641112327576,
+      "step": 197
+    },
+    {
+      "epoch": 1.875,
+      "grad_norm": 4.468769998980834,
+      "learning_rate": 6.349206349206348e-09,
+      "logits/chosen": -1.4640628099441528,
+      "logits/rejected": -1.5838810205459595,
+      "logps/chosen": -238.00106811523438,
+      "logps/rejected": -104.63729095458984,
+      "loss": 0.3595,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5341485142707825,
+      "rewards/margins": 1.1648151874542236,
+      "rewards/rejected": -0.6306666135787964,
+      "step": 198
+    },
+    {
+      "epoch": 1.884469696969697,
+      "grad_norm": 5.080896081269075,
+      "learning_rate": 5.820105820105819e-09,
+      "logits/chosen": -1.5636723041534424,
+      "logits/rejected": -1.658831238746643,
+      "logps/chosen": -203.7349395751953,
+      "logps/rejected": -110.94892883300781,
+      "loss": 0.3676,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.3449416756629944,
+      "rewards/margins": 0.7937332987785339,
+      "rewards/rejected": -0.44879165291786194,
+      "step": 199
+    },
+    {
+      "epoch": 1.893939393939394,
+      "grad_norm": 5.280192220580751,
+      "learning_rate": 5.29100529100529e-09,
+      "logits/chosen": -1.5569162368774414,
+      "logits/rejected": -1.6692053079605103,
+      "logps/chosen": -310.94293212890625,
+      "logps/rejected": -180.21963500976562,
+      "loss": 0.4127,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6343954801559448,
+      "rewards/margins": 1.3330564498901367,
+      "rewards/rejected": -0.6986608505249023,
+      "step": 200
+    },
+    {
+      "epoch": 1.9034090909090908,
+      "grad_norm": 4.3144901310959956,
+      "learning_rate": 4.761904761904761e-09,
+      "logits/chosen": -1.5700764656066895,
+      "logits/rejected": -1.5398668050765991,
+      "logps/chosen": -123.7179946899414,
+      "logps/rejected": -168.89608764648438,
+      "loss": 0.3673,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.18818268179893494,
+      "rewards/margins": 0.4049772620201111,
+      "rewards/rejected": -0.21679461002349854,
+      "step": 201
+    },
+    {
+      "epoch": 1.9128787878787878,
+      "grad_norm": 4.809421227753802,
+      "learning_rate": 4.232804232804232e-09,
+      "logits/chosen": -1.5965473651885986,
+      "logits/rejected": -1.7248961925506592,
+      "logps/chosen": -306.676025390625,
+      "logps/rejected": -186.1800994873047,
+      "loss": 0.363,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": 0.401868999004364,
+      "rewards/margins": 0.8128830194473267,
+      "rewards/rejected": -0.41101402044296265,
+      "step": 202
+    },
+    {
+      "epoch": 1.9223484848484849,
+      "grad_norm": 4.639189256023351,
+      "learning_rate": 3.703703703703703e-09,
+      "logits/chosen": -1.5983052253723145,
+      "logits/rejected": -1.6595712900161743,
+      "logps/chosen": -307.19183349609375,
+      "logps/rejected": -195.84725952148438,
+      "loss": 0.3743,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.6213973760604858,
+      "rewards/margins": 1.057089924812317,
+      "rewards/rejected": -0.4356924891471863,
+      "step": 203
+    },
+    {
+      "epoch": 1.9318181818181817,
+      "grad_norm": 5.728537848527359,
+      "learning_rate": 3.174603174603174e-09,
+      "logits/chosen": -1.5698726177215576,
+      "logits/rejected": -1.6646344661712646,
+      "logps/chosen": -269.0412292480469,
+      "logps/rejected": -137.50003051757812,
+      "loss": 0.3975,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5319095849990845,
+      "rewards/margins": 1.2130988836288452,
+      "rewards/rejected": -0.6811891794204712,
+      "step": 204
+    },
+    {
+      "epoch": 1.941287878787879,
+      "grad_norm": 5.416585887060536,
+      "learning_rate": 2.645502645502645e-09,
+      "logits/chosen": -1.6159827709197998,
+      "logits/rejected": -1.8529045581817627,
+      "logps/chosen": -324.2301025390625,
+      "logps/rejected": -169.9268035888672,
+      "loss": 0.4161,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": 0.44274115562438965,
+      "rewards/margins": 0.865648090839386,
+      "rewards/rejected": -0.4229070544242859,
+      "step": 205
+    },
+    {
+      "epoch": 1.9507575757575757,
+      "grad_norm": 5.021756713871844,
+      "learning_rate": 2.116402116402116e-09,
+      "logits/chosen": -1.6490414142608643,
+      "logits/rejected": -1.701080083847046,
+      "logps/chosen": -223.4647979736328,
+      "logps/rejected": -112.0922622680664,
+      "loss": 0.4041,
+      "rewards/accuracies": 0.9333332777023315,
+      "rewards/chosen": 0.25973638892173767,
+      "rewards/margins": 0.5601609349250793,
+      "rewards/rejected": -0.3004245162010193,
+      "step": 206
+    },
+    {
+      "epoch": 1.9602272727272727,
+      "grad_norm": 5.226300428110887,
+      "learning_rate": 1.587301587301587e-09,
+      "logits/chosen": -1.6259441375732422,
+      "logits/rejected": -1.6718814373016357,
+      "logps/chosen": -314.91583251953125,
+      "logps/rejected": -182.3265838623047,
+      "loss": 0.3799,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.5016330480575562,
+      "rewards/margins": 1.0548944473266602,
+      "rewards/rejected": -0.553261399269104,
+      "step": 207
+    },
+    {
+      "epoch": 1.9696969696969697,
+      "grad_norm": 4.5059277056393725,
+      "learning_rate": 1.058201058201058e-09,
+      "logits/chosen": -1.6556800603866577,
+      "logits/rejected": -1.7395381927490234,
+      "logps/chosen": -346.4697570800781,
+      "logps/rejected": -191.90931701660156,
+      "loss": 0.3702,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8226908445358276,
+      "rewards/margins": 1.620936393737793,
+      "rewards/rejected": -0.7982456684112549,
+      "step": 208
+    },
+    {
+      "epoch": 1.9791666666666665,
+      "grad_norm": 4.466235222468724,
+      "learning_rate": 5.29100529100529e-10,
+      "logits/chosen": -1.5888211727142334,
+      "logits/rejected": -1.6132078170776367,
+      "logps/chosen": -215.74612426757812,
+      "logps/rejected": -148.16468811035156,
+      "loss": 0.3622,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.3967617154121399,
+      "rewards/margins": 0.8830636143684387,
+      "rewards/rejected": -0.48630183935165405,
+      "step": 209
+    },
+    {
+      "epoch": 1.9886363636363638,
+      "grad_norm": 4.940010839228651,
+      "learning_rate": 0.0,
+      "logits/chosen": -1.6120548248291016,
+      "logits/rejected": -1.675747275352478,
+      "logps/chosen": -285.6683654785156,
+      "logps/rejected": -142.8597869873047,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": 0.4850710332393646,
+      "rewards/margins": 1.1287145614624023,
+      "rewards/rejected": -0.6436434984207153,
+      "step": 210
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 210,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}