trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 4.99738356881214,
  "eval_steps": 100,
  "global_step": 4775,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0010465724751439038,
      "grad_norm": 1.5610575235354534,
      "learning_rate": 1.0460251046025104e-07,
      "logits/chosen": -9.375,
      "logits/rejected": -9.5,
      "logps/chosen": -448.0,
      "logps/rejected": -398.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.010465724751439037,
      "grad_norm": 1.349006948243697,
      "learning_rate": 1.0460251046025104e-06,
      "logits/chosen": -9.5,
      "logits/rejected": -9.625,
      "logps/chosen": -296.0,
      "logps/rejected": -264.0,
      "loss": 0.6925,
      "rewards/accuracies": 0.2083333283662796,
      "rewards/chosen": 0.0012054443359375,
      "rewards/margins": 0.000579833984375,
      "rewards/rejected": 0.0006256103515625,
      "step": 10
    },
    {
      "epoch": 0.020931449502878074,
      "grad_norm": 1.5264399374921611,
      "learning_rate": 2.092050209205021e-06,
      "logits/chosen": -9.5,
      "logits/rejected": -9.4375,
      "logps/chosen": -324.0,
      "logps/rejected": -310.0,
      "loss": 0.6918,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": 0.00299072265625,
      "rewards/margins": 0.003204345703125,
      "rewards/rejected": -0.00020885467529296875,
      "step": 20
    },
    {
      "epoch": 0.03139717425431711,
      "grad_norm": 1.583736957016309,
      "learning_rate": 3.1380753138075313e-06,
      "logits/chosen": -9.6875,
      "logits/rejected": -9.5,
      "logps/chosen": -314.0,
      "logps/rejected": -310.0,
      "loss": 0.6908,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": 0.00579833984375,
      "rewards/margins": 0.001678466796875,
      "rewards/rejected": 0.00408935546875,
      "step": 30
    },
    {
      "epoch": 0.04186289900575615,
      "grad_norm": 1.4892011169858126,
      "learning_rate": 4.184100418410042e-06,
      "logits/chosen": -9.5625,
      "logits/rejected": -9.8125,
      "logps/chosen": -334.0,
      "logps/rejected": -308.0,
      "loss": 0.6868,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": 0.00885009765625,
      "rewards/margins": 0.0108642578125,
      "rewards/rejected": -0.0019683837890625,
      "step": 40
    },
    {
      "epoch": 0.052328623757195186,
      "grad_norm": 1.3871844092880496,
      "learning_rate": 5.230125523012553e-06,
      "logits/chosen": -9.5625,
      "logits/rejected": -9.5,
      "logps/chosen": -316.0,
      "logps/rejected": -270.0,
      "loss": 0.6818,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.040771484375,
      "rewards/margins": 0.0247802734375,
      "rewards/rejected": -0.0654296875,
      "step": 50
    },
    {
      "epoch": 0.06279434850863422,
      "grad_norm": 1.5257182239483598,
      "learning_rate": 6.276150627615063e-06,
      "logits/chosen": -9.0625,
      "logits/rejected": -9.0,
      "logps/chosen": -364.0,
      "logps/rejected": -324.0,
      "loss": 0.6721,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.11474609375,
      "rewards/margins": 0.049072265625,
      "rewards/rejected": -0.1640625,
      "step": 60
    },
    {
      "epoch": 0.07326007326007326,
      "grad_norm": 1.5659951271442103,
      "learning_rate": 7.3221757322175736e-06,
      "logits/chosen": -9.875,
      "logits/rejected": -9.5,
      "logps/chosen": -326.0,
      "logps/rejected": -320.0,
      "loss": 0.6597,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.2734375,
      "rewards/margins": 0.0859375,
      "rewards/rejected": -0.359375,
      "step": 70
    },
    {
      "epoch": 0.0837257980115123,
      "grad_norm": 1.8241831527626855,
      "learning_rate": 8.368200836820084e-06,
      "logits/chosen": -9.6875,
      "logits/rejected": -10.125,
      "logps/chosen": -360.0,
      "logps/rejected": -328.0,
      "loss": 0.6419,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.32421875,
      "rewards/margins": 0.166015625,
      "rewards/rejected": -0.490234375,
      "step": 80
    },
    {
      "epoch": 0.09419152276295134,
      "grad_norm": 1.6581722991160308,
      "learning_rate": 9.414225941422594e-06,
      "logits/chosen": -9.9375,
      "logits/rejected": -9.9375,
      "logps/chosen": -348.0,
      "logps/rejected": -364.0,
      "loss": 0.6405,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.6015625,
      "rewards/margins": 0.0888671875,
      "rewards/rejected": -0.69140625,
      "step": 90
    },
    {
      "epoch": 0.10465724751439037,
      "grad_norm": 1.9270010320276483,
      "learning_rate": 1.0460251046025105e-05,
      "logits/chosen": -8.75,
      "logits/rejected": -8.625,
      "logps/chosen": -416.0,
      "logps/rejected": -402.0,
      "loss": 0.6232,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6875,
      "rewards/margins": 0.25,
      "rewards/rejected": -0.9375,
      "step": 100
    },
    {
      "epoch": 0.10465724751439037,
      "eval_logits/chosen": -8.5,
      "eval_logits/rejected": -8.25,
      "eval_logps/chosen": -374.0,
      "eval_logps/rejected": -368.0,
      "eval_loss": 0.6285156011581421,
      "eval_rewards/accuracies": 0.666015625,
      "eval_rewards/chosen": -0.60546875,
      "eval_rewards/margins": 0.2177734375,
      "eval_rewards/rejected": -0.82421875,
      "eval_runtime": 53.3726,
      "eval_samples_per_second": 37.472,
      "eval_steps_per_second": 0.6,
      "step": 100
    },
    {
      "epoch": 0.1151229722658294,
      "grad_norm": 1.8649818778865919,
      "learning_rate": 1.1506276150627615e-05,
      "logits/chosen": -8.4375,
      "logits/rejected": -8.3125,
      "logps/chosen": -352.0,
      "logps/rejected": -360.0,
      "loss": 0.6325,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.703125,
      "rewards/margins": 0.21484375,
      "rewards/rejected": -0.91796875,
      "step": 110
    },
    {
      "epoch": 0.12558869701726844,
      "grad_norm": 3.1874202308933772,
      "learning_rate": 1.2552301255230125e-05,
      "logits/chosen": -9.0,
      "logits/rejected": -8.6875,
      "logps/chosen": -386.0,
      "logps/rejected": -426.0,
      "loss": 0.6012,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.78125,
      "rewards/margins": 0.2734375,
      "rewards/rejected": -1.0546875,
      "step": 120
    },
    {
      "epoch": 0.1360544217687075,
      "grad_norm": 2.8031812157693743,
      "learning_rate": 1.3598326359832636e-05,
      "logits/chosen": -8.75,
      "logits/rejected": -8.5,
      "logps/chosen": -452.0,
      "logps/rejected": -452.0,
      "loss": 0.6069,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.21875,
      "rewards/margins": 0.435546875,
      "rewards/rejected": -1.65625,
      "step": 130
    },
    {
      "epoch": 0.14652014652014653,
      "grad_norm": 2.197923048367805,
      "learning_rate": 1.4644351464435147e-05,
      "logits/chosen": -10.1875,
      "logits/rejected": -10.0625,
      "logps/chosen": -400.0,
      "logps/rejected": -412.0,
      "loss": 0.5996,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.65625,
      "rewards/margins": 0.357421875,
      "rewards/rejected": -1.015625,
      "step": 140
    },
    {
      "epoch": 0.15698587127158556,
      "grad_norm": 1.9425267890042368,
      "learning_rate": 1.5690376569037658e-05,
      "logits/chosen": -11.4375,
      "logits/rejected": -11.0625,
      "logps/chosen": -418.0,
      "logps/rejected": -402.0,
      "loss": 0.6035,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8359375,
      "rewards/margins": 0.373046875,
      "rewards/rejected": -1.2109375,
      "step": 150
    },
    {
      "epoch": 0.1674515960230246,
      "grad_norm": 2.6369764941319116,
      "learning_rate": 1.6736401673640167e-05,
      "logits/chosen": -11.625,
      "logits/rejected": -11.375,
      "logps/chosen": -440.0,
      "logps/rejected": -418.0,
      "loss": 0.5842,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.92578125,
      "rewards/margins": 0.43359375,
      "rewards/rejected": -1.359375,
      "step": 160
    },
    {
      "epoch": 0.17791732077446362,
      "grad_norm": 2.206623859417659,
      "learning_rate": 1.7782426778242676e-05,
      "logits/chosen": -10.0625,
      "logits/rejected": -9.5625,
      "logps/chosen": -470.0,
      "logps/rejected": -480.0,
      "loss": 0.5908,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 0.55078125,
      "rewards/rejected": -2.109375,
      "step": 170
    },
    {
      "epoch": 0.18838304552590268,
      "grad_norm": 2.3093801072587725,
      "learning_rate": 1.882845188284519e-05,
      "logits/chosen": -6.21875,
      "logits/rejected": -5.53125,
      "logps/chosen": -426.0,
      "logps/rejected": -470.0,
      "loss": 0.5717,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 0.5546875,
      "rewards/rejected": -1.984375,
      "step": 180
    },
    {
      "epoch": 0.1988487702773417,
      "grad_norm": 2.4317513580230186,
      "learning_rate": 1.9874476987447698e-05,
      "logits/chosen": -4.8125,
      "logits/rejected": -4.4375,
      "logps/chosen": -434.0,
      "logps/rejected": -480.0,
      "loss": 0.5755,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 0.4453125,
      "rewards/rejected": -2.03125,
      "step": 190
    },
    {
      "epoch": 0.20931449502878074,
      "grad_norm": 2.5804173107312147,
      "learning_rate": 2.092050209205021e-05,
      "logits/chosen": -6.9375,
      "logits/rejected": -6.4375,
      "logps/chosen": -516.0,
      "logps/rejected": -490.0,
      "loss": 0.5729,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5625,
      "rewards/margins": 0.51171875,
      "rewards/rejected": -2.078125,
      "step": 200
    },
    {
      "epoch": 0.20931449502878074,
      "eval_logits/chosen": -8.4375,
      "eval_logits/rejected": -7.96875,
      "eval_logps/chosen": -478.0,
      "eval_logps/rejected": -498.0,
      "eval_loss": 0.595703125,
      "eval_rewards/accuracies": 0.69921875,
      "eval_rewards/chosen": -1.6328125,
      "eval_rewards/margins": 0.4765625,
      "eval_rewards/rejected": -2.109375,
      "eval_runtime": 52.7631,
      "eval_samples_per_second": 37.905,
      "eval_steps_per_second": 0.606,
      "step": 200
    },
    {
      "epoch": 0.21978021978021978,
      "grad_norm": 2.028055345355448,
      "learning_rate": 2.196652719665272e-05,
      "logits/chosen": -9.25,
      "logits/rejected": -8.8125,
      "logps/chosen": -504.0,
      "logps/rejected": -512.0,
      "loss": 0.616,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.765625,
      "rewards/margins": 0.283203125,
      "rewards/rejected": -2.046875,
      "step": 210
    },
    {
      "epoch": 0.2302459445316588,
      "grad_norm": 2.4926314967465726,
      "learning_rate": 2.301255230125523e-05,
      "logits/chosen": -8.1875,
      "logits/rejected": -7.90625,
      "logps/chosen": -516.0,
      "logps/rejected": -488.0,
      "loss": 0.5839,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 0.470703125,
      "rewards/rejected": -2.328125,
      "step": 220
    },
    {
      "epoch": 0.24071166928309787,
      "grad_norm": 2.2773378614378026,
      "learning_rate": 2.405857740585774e-05,
      "logits/chosen": -7.4375,
      "logits/rejected": -7.0,
      "logps/chosen": -502.0,
      "logps/rejected": -512.0,
      "loss": 0.5985,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 0.400390625,
      "rewards/rejected": -2.109375,
      "step": 230
    },
    {
      "epoch": 0.25117739403453687,
      "grad_norm": 2.1409273439247754,
      "learning_rate": 2.510460251046025e-05,
      "logits/chosen": -9.8125,
      "logits/rejected": -9.5,
      "logps/chosen": -472.0,
      "logps/rejected": -492.0,
      "loss": 0.5804,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 0.439453125,
      "rewards/rejected": -2.0625,
      "step": 240
    },
    {
      "epoch": 0.2616431187859759,
      "grad_norm": 2.8417736797719093,
      "learning_rate": 2.615062761506276e-05,
      "logits/chosen": -7.5,
      "logits/rejected": -7.0,
      "logps/chosen": -552.0,
      "logps/rejected": -572.0,
      "loss": 0.5424,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.296875,
      "rewards/margins": 0.5546875,
      "rewards/rejected": -2.84375,
      "step": 250
    },
    {
      "epoch": 0.272108843537415,
      "grad_norm": 2.1201241469645438,
      "learning_rate": 2.7196652719665273e-05,
      "logits/chosen": -9.25,
      "logits/rejected": -8.8125,
      "logps/chosen": -572.0,
      "logps/rejected": -596.0,
      "loss": 0.5807,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -2.40625,
      "rewards/margins": 0.42578125,
      "rewards/rejected": -2.828125,
      "step": 260
    },
    {
      "epoch": 0.282574568288854,
      "grad_norm": 2.809629729626212,
      "learning_rate": 2.824267782426778e-05,
      "logits/chosen": -6.875,
      "logits/rejected": -6.3125,
      "logps/chosen": -544.0,
      "logps/rejected": -528.0,
      "loss": 0.5719,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 0.4765625,
      "rewards/rejected": -2.421875,
      "step": 270
    },
    {
      "epoch": 0.29304029304029305,
      "grad_norm": 2.0280888428773807,
      "learning_rate": 2.9288702928870294e-05,
      "logits/chosen": -5.6875,
      "logits/rejected": -5.125,
      "logps/chosen": -494.0,
      "logps/rejected": -510.0,
      "loss": 0.5969,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5,
      "rewards/margins": 0.443359375,
      "rewards/rejected": -1.9453125,
      "step": 280
    },
    {
      "epoch": 0.3035060177917321,
      "grad_norm": 2.5524631342772692,
      "learning_rate": 3.0334728033472803e-05,
      "logits/chosen": -6.65625,
      "logits/rejected": -6.25,
      "logps/chosen": -486.0,
      "logps/rejected": -468.0,
      "loss": 0.6021,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.6875,
      "rewards/margins": 0.3515625,
      "rewards/rejected": -2.03125,
      "step": 290
    },
    {
      "epoch": 0.3139717425431711,
      "grad_norm": 2.353526727859833,
      "learning_rate": 3.1380753138075316e-05,
      "logits/chosen": -6.8125,
      "logits/rejected": -6.65625,
      "logps/chosen": -544.0,
      "logps/rejected": -564.0,
      "loss": 0.6122,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -2.328125,
      "rewards/margins": 0.375,
      "rewards/rejected": -2.703125,
      "step": 300
    },
    {
      "epoch": 0.3139717425431711,
      "eval_logits/chosen": -6.15625,
      "eval_logits/rejected": -5.59375,
      "eval_logps/chosen": -474.0,
      "eval_logps/rejected": -496.0,
      "eval_loss": 0.5750937461853027,
      "eval_rewards/accuracies": 0.712890625,
      "eval_rewards/chosen": -1.6015625,
      "eval_rewards/margins": 0.5,
      "eval_rewards/rejected": -2.109375,
      "eval_runtime": 52.7701,
      "eval_samples_per_second": 37.9,
      "eval_steps_per_second": 0.606,
      "step": 300
    },
    {
      "epoch": 0.32443746729461015,
      "grad_norm": 2.534963819166362,
      "learning_rate": 3.2426778242677825e-05,
      "logits/chosen": -7.0,
      "logits/rejected": -5.84375,
      "logps/chosen": -492.0,
      "logps/rejected": -456.0,
      "loss": 0.588,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 0.494140625,
      "rewards/rejected": -2.0625,
      "step": 310
    },
    {
      "epoch": 0.3349031920460492,
      "grad_norm": 1.9350591762386424,
      "learning_rate": 3.3472803347280334e-05,
      "logits/chosen": -6.53125,
      "logits/rejected": -6.0625,
      "logps/chosen": -476.0,
      "logps/rejected": -516.0,
      "loss": 0.6021,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 0.50390625,
      "rewards/rejected": -2.390625,
      "step": 320
    },
    {
      "epoch": 0.3453689167974882,
      "grad_norm": 2.255741471296445,
      "learning_rate": 3.4518828451882844e-05,
      "logits/chosen": -7.5,
      "logits/rejected": -7.09375,
      "logps/chosen": -528.0,
      "logps/rejected": -536.0,
      "loss": 0.582,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.296875,
      "rewards/margins": 0.455078125,
      "rewards/rejected": -2.75,
      "step": 330
    },
    {
      "epoch": 0.35583464154892724,
      "grad_norm": 2.829695974746096,
      "learning_rate": 3.556485355648535e-05,
      "logits/chosen": -7.65625,
      "logits/rejected": -7.0625,
      "logps/chosen": -524.0,
      "logps/rejected": -520.0,
      "loss": 0.5962,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -2.15625,
      "rewards/margins": 0.384765625,
      "rewards/rejected": -2.53125,
      "step": 340
    },
    {
      "epoch": 0.3663003663003663,
      "grad_norm": 1.9387450136987523,
      "learning_rate": 3.661087866108787e-05,
      "logits/chosen": -5.3125,
      "logits/rejected": -4.59375,
      "logps/chosen": -612.0,
      "logps/rejected": -608.0,
      "loss": 0.5795,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -2.703125,
      "rewards/margins": 0.54296875,
      "rewards/rejected": -3.25,
      "step": 350
    },
    {
      "epoch": 0.37676609105180536,
      "grad_norm": 2.4596243424669755,
      "learning_rate": 3.765690376569038e-05,
      "logits/chosen": -5.3125,
      "logits/rejected": -4.59375,
      "logps/chosen": -540.0,
      "logps/rejected": -560.0,
      "loss": 0.5594,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.21875,
      "rewards/margins": 0.53515625,
      "rewards/rejected": -2.765625,
      "step": 360
    },
    {
      "epoch": 0.3872318158032444,
      "grad_norm": 2.38807244727826,
      "learning_rate": 3.870292887029289e-05,
      "logits/chosen": -6.46875,
      "logits/rejected": -6.09375,
      "logps/chosen": -588.0,
      "logps/rejected": -596.0,
      "loss": 0.5757,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -2.796875,
      "rewards/margins": 0.466796875,
      "rewards/rejected": -3.265625,
      "step": 370
    },
    {
      "epoch": 0.3976975405546834,
      "grad_norm": 2.4861103037966816,
      "learning_rate": 3.9748953974895396e-05,
      "logits/chosen": -6.125,
      "logits/rejected": -5.65625,
      "logps/chosen": -568.0,
      "logps/rejected": -600.0,
      "loss": 0.5829,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -2.71875,
      "rewards/margins": 0.4609375,
      "rewards/rejected": -3.1875,
      "step": 380
    },
    {
      "epoch": 0.40816326530612246,
      "grad_norm": 2.1200670617456407,
      "learning_rate": 4.0794979079497905e-05,
      "logits/chosen": -6.28125,
      "logits/rejected": -5.6875,
      "logps/chosen": -516.0,
      "logps/rejected": -528.0,
      "loss": 0.6082,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.90625,
      "rewards/margins": 0.3671875,
      "rewards/rejected": -2.28125,
      "step": 390
    },
    {
      "epoch": 0.4186289900575615,
      "grad_norm": 3.353244167214107,
      "learning_rate": 4.184100418410042e-05,
      "logits/chosen": -6.21875,
      "logits/rejected": -5.4375,
      "logps/chosen": -510.0,
      "logps/rejected": -536.0,
      "loss": 0.5905,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.953125,
      "rewards/margins": 0.455078125,
      "rewards/rejected": -2.40625,
      "step": 400
    },
    {
      "epoch": 0.4186289900575615,
      "eval_logits/chosen": -6.34375,
      "eval_logits/rejected": -5.5625,
      "eval_logps/chosen": -478.0,
      "eval_logps/rejected": -504.0,
      "eval_loss": 0.599359393119812,
      "eval_rewards/accuracies": 0.66796875,
      "eval_rewards/chosen": -1.6328125,
      "eval_rewards/margins": 0.55078125,
      "eval_rewards/rejected": -2.1875,
      "eval_runtime": 52.7697,
      "eval_samples_per_second": 37.901,
      "eval_steps_per_second": 0.606,
      "step": 400
    },
    {
      "epoch": 0.4290947148090005,
      "grad_norm": 1.9400560513665543,
      "learning_rate": 4.288702928870293e-05,
      "logits/chosen": -5.875,
      "logits/rejected": -5.0625,
      "logps/chosen": -532.0,
      "logps/rejected": -540.0,
      "loss": 0.604,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -2.109375,
      "rewards/margins": 0.515625,
      "rewards/rejected": -2.625,
      "step": 410
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 1.801312683220209,
      "learning_rate": 4.393305439330544e-05,
      "logits/chosen": -5.0,
      "logits/rejected": -4.125,
      "logps/chosen": -564.0,
      "logps/rejected": -560.0,
      "loss": 0.6109,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -2.328125,
      "rewards/margins": 0.42578125,
      "rewards/rejected": -2.75,
      "step": 420
    },
    {
      "epoch": 0.4500261643118786,
      "grad_norm": 1.7147949043618933,
      "learning_rate": 4.497907949790795e-05,
      "logits/chosen": -6.125,
      "logits/rejected": -5.5625,
      "logps/chosen": -478.0,
      "logps/rejected": -502.0,
      "loss": 0.6254,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 0.369140625,
      "rewards/rejected": -2.15625,
      "step": 430
    },
    {
      "epoch": 0.4604918890633176,
      "grad_norm": 2.796041254164824,
      "learning_rate": 4.602510460251046e-05,
      "logits/chosen": -7.71875,
      "logits/rejected": -7.3125,
      "logps/chosen": -528.0,
      "logps/rejected": -504.0,
      "loss": 0.611,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.90625,
      "rewards/margins": 0.392578125,
      "rewards/rejected": -2.296875,
      "step": 440
    },
    {
      "epoch": 0.47095761381475665,
      "grad_norm": 2.1381334456656744,
      "learning_rate": 4.7071129707112974e-05,
      "logits/chosen": -8.3125,
      "logits/rejected": -7.96875,
      "logps/chosen": -520.0,
      "logps/rejected": -556.0,
      "loss": 0.5771,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.875,
      "rewards/margins": 0.58984375,
      "rewards/rejected": -2.46875,
      "step": 450
    },
    {
      "epoch": 0.48142333856619574,
      "grad_norm": 1.8842575828374497,
      "learning_rate": 4.811715481171548e-05,
      "logits/chosen": -8.5,
      "logits/rejected": -8.375,
      "logps/chosen": -532.0,
      "logps/rejected": -552.0,
      "loss": 0.5877,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -2.3125,
      "rewards/margins": 0.5078125,
      "rewards/rejected": -2.828125,
      "step": 460
    },
    {
      "epoch": 0.49188906331763477,
      "grad_norm": 2.295050225174663,
      "learning_rate": 4.916317991631799e-05,
      "logits/chosen": -9.3125,
      "logits/rejected": -8.8125,
      "logps/chosen": -500.0,
      "logps/rejected": -508.0,
      "loss": 0.5301,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 0.62109375,
      "rewards/rejected": -2.359375,
      "step": 470
    },
    {
      "epoch": 0.5023547880690737,
      "grad_norm": 2.159411861590043,
      "learning_rate": 4.9999973273692244e-05,
      "logits/chosen": -9.625,
      "logits/rejected": -9.25,
      "logps/chosen": -532.0,
      "logps/rejected": -576.0,
      "loss": 0.5703,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.203125,
      "rewards/margins": 0.734375,
      "rewards/rejected": -2.9375,
      "step": 480
    },
    {
      "epoch": 0.5128205128205128,
      "grad_norm": 1.8524703002374596,
      "learning_rate": 4.9999037858921e-05,
      "logits/chosen": -7.875,
      "logits/rejected": -7.125,
      "logps/chosen": -620.0,
      "logps/rejected": -620.0,
      "loss": 0.5705,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.546875,
      "rewards/margins": 0.7890625,
      "rewards/rejected": -3.34375,
      "step": 490
    },
    {
      "epoch": 0.5232862375719518,
      "grad_norm": 2.3441132459203655,
      "learning_rate": 4.9996766185905256e-05,
      "logits/chosen": -6.75,
      "logits/rejected": -6.125,
      "logps/chosen": -490.0,
      "logps/rejected": -496.0,
      "loss": 0.5781,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.90625,
      "rewards/margins": 0.53125,
      "rewards/rejected": -2.4375,
      "step": 500
    },
    {
      "epoch": 0.5232862375719518,
      "eval_logits/chosen": -6.84375,
      "eval_logits/rejected": -6.0625,
      "eval_logps/chosen": -486.0,
      "eval_logps/rejected": -512.0,
      "eval_loss": 0.5763593912124634,
      "eval_rewards/accuracies": 0.681640625,
      "eval_rewards/chosen": -1.71875,
      "eval_rewards/margins": 0.55859375,
      "eval_rewards/rejected": -2.265625,
      "eval_runtime": 52.7734,
      "eval_samples_per_second": 37.898,
      "eval_steps_per_second": 0.606,
      "step": 500
    },
    {
      "epoch": 0.533751962323391,
      "grad_norm": 1.6521111958327657,
      "learning_rate": 4.999315837607136e-05,
      "logits/chosen": -7.03125,
      "logits/rejected": -6.25,
      "logps/chosen": -524.0,
      "logps/rejected": -556.0,
      "loss": 0.572,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 0.68359375,
      "rewards/rejected": -2.609375,
      "step": 510
    },
    {
      "epoch": 0.54421768707483,
      "grad_norm": 2.124354025750924,
      "learning_rate": 4.998821462226535e-05,
      "logits/chosen": -4.5,
      "logits/rejected": -3.90625,
      "logps/chosen": -672.0,
      "logps/rejected": -688.0,
      "loss": 0.5989,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 0.498046875,
      "rewards/rejected": -4.03125,
      "step": 520
    },
    {
      "epoch": 0.554683411826269,
      "grad_norm": 2.1800165538121425,
      "learning_rate": 4.998193518874268e-05,
      "logits/chosen": -7.125,
      "logits/rejected": -6.46875,
      "logps/chosen": -640.0,
      "logps/rejected": -668.0,
      "loss": 0.6098,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -3.234375,
      "rewards/margins": 0.57421875,
      "rewards/rejected": -3.8125,
      "step": 530
    },
    {
      "epoch": 0.565149136577708,
      "grad_norm": 1.8923017590968032,
      "learning_rate": 4.997432041115405e-05,
      "logits/chosen": -8.5,
      "logits/rejected": -7.8125,
      "logps/chosen": -560.0,
      "logps/rejected": -548.0,
      "loss": 0.5872,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.15625,
      "rewards/margins": 0.4296875,
      "rewards/rejected": -2.578125,
      "step": 540
    },
    {
      "epoch": 0.5756148613291471,
      "grad_norm": 1.507962725186336,
      "learning_rate": 4.9965370696527504e-05,
      "logits/chosen": -6.875,
      "logits/rejected": -6.0,
      "logps/chosen": -620.0,
      "logps/rejected": -616.0,
      "loss": 0.6074,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.109375,
      "rewards/margins": 0.55078125,
      "rewards/rejected": -3.65625,
      "step": 550
    },
    {
      "epoch": 0.5860805860805861,
      "grad_norm": 1.91935882186746,
      "learning_rate": 4.9955086523246655e-05,
      "logits/chosen": -7.84375,
      "logits/rejected": -7.0625,
      "logps/chosen": -560.0,
      "logps/rejected": -576.0,
      "loss": 0.5955,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -2.328125,
      "rewards/margins": 0.5703125,
      "rewards/rejected": -2.90625,
      "step": 560
    },
    {
      "epoch": 0.5965463108320251,
      "grad_norm": 1.8238249534958182,
      "learning_rate": 4.99434684410251e-05,
      "logits/chosen": -8.4375,
      "logits/rejected": -7.78125,
      "logps/chosen": -552.0,
      "logps/rejected": -572.0,
      "loss": 0.5758,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -2.25,
      "rewards/margins": 0.55078125,
      "rewards/rejected": -2.796875,
      "step": 570
    },
    {
      "epoch": 0.6070120355834642,
      "grad_norm": 2.0134942953537562,
      "learning_rate": 4.9930517070877066e-05,
      "logits/chosen": -7.90625,
      "logits/rejected": -7.21875,
      "logps/chosen": -632.0,
      "logps/rejected": -644.0,
      "loss": 0.6037,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.390625,
      "rewards/margins": 0.71484375,
      "rewards/rejected": -4.09375,
      "step": 580
    },
    {
      "epoch": 0.6174777603349032,
      "grad_norm": 1.8626820784967801,
      "learning_rate": 4.991623310508422e-05,
      "logits/chosen": -8.5,
      "logits/rejected": -7.65625,
      "logps/chosen": -576.0,
      "logps/rejected": -616.0,
      "loss": 0.5652,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -2.546875,
      "rewards/margins": 0.6796875,
      "rewards/rejected": -3.234375,
      "step": 590
    },
    {
      "epoch": 0.6279434850863422,
      "grad_norm": 2.3196683948602286,
      "learning_rate": 4.990061730715862e-05,
      "logits/chosen": -5.28125,
      "logits/rejected": -4.375,
      "logps/chosen": -644.0,
      "logps/rejected": -692.0,
      "loss": 0.5356,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.34375,
      "rewards/margins": 0.75,
      "rewards/rejected": -4.09375,
      "step": 600
    },
    {
      "epoch": 0.6279434850863422,
      "eval_logits/chosen": -4.25,
      "eval_logits/rejected": -3.328125,
      "eval_logps/chosen": -704.0,
      "eval_logps/rejected": -744.0,
      "eval_loss": 0.5830625295639038,
      "eval_rewards/accuracies": 0.669921875,
      "eval_rewards/chosen": -3.890625,
      "eval_rewards/margins": 0.6796875,
      "eval_rewards/rejected": -4.5625,
      "eval_runtime": 52.7702,
      "eval_samples_per_second": 37.9,
      "eval_steps_per_second": 0.606,
      "step": 600
    },
    {
      "epoch": 0.6384092098377813,
      "grad_norm": 1.969214676284523,
      "learning_rate": 4.9883670511801914e-05,
      "logits/chosen": -5.03125,
      "logits/rejected": -3.90625,
      "logps/chosen": -700.0,
      "logps/rejected": -688.0,
      "loss": 0.6028,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -3.4375,
      "rewards/margins": 0.67578125,
      "rewards/rejected": -4.09375,
      "step": 610
    },
    {
      "epoch": 0.6488749345892203,
      "grad_norm": 1.8292251328731444,
      "learning_rate": 4.98653936248608e-05,
      "logits/chosen": -5.46875,
      "logits/rejected": -4.75,
      "logps/chosen": -556.0,
      "logps/rejected": -572.0,
      "loss": 0.5732,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.4375,
      "rewards/margins": 0.6640625,
      "rewards/rejected": -3.09375,
      "step": 620
    },
    {
      "epoch": 0.6593406593406593,
      "grad_norm": 2.0559585908544658,
      "learning_rate": 4.9845787623278495e-05,
      "logits/chosen": -6.09375,
      "logits/rejected": -5.34375,
      "logps/chosen": -580.0,
      "logps/rejected": -592.0,
      "loss": 0.559,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.640625,
      "rewards/margins": 0.65234375,
      "rewards/rejected": -3.296875,
      "step": 630
    },
    {
      "epoch": 0.6698063840920984,
      "grad_norm": 2.367176900894739,
      "learning_rate": 4.982485355504258e-05,
      "logits/chosen": -5.34375,
      "logits/rejected": -4.375,
      "logps/chosen": -616.0,
      "logps/rejected": -632.0,
      "loss": 0.5808,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -3.03125,
      "rewards/margins": 0.68359375,
      "rewards/rejected": -3.71875,
      "step": 640
    },
    {
      "epoch": 0.6802721088435374,
      "grad_norm": 1.9017110444929817,
      "learning_rate": 4.9802592539128984e-05,
      "logits/chosen": -6.59375,
      "logits/rejected": -5.875,
      "logps/chosen": -544.0,
      "logps/rejected": -596.0,
      "loss": 0.5699,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.390625,
      "rewards/margins": 0.55078125,
      "rewards/rejected": -2.9375,
      "step": 650
    },
    {
      "epoch": 0.6907378335949764,
      "grad_norm": 1.5844136116169825,
      "learning_rate": 4.977900576544212e-05,
      "logits/chosen": -5.15625,
      "logits/rejected": -4.03125,
      "logps/chosen": -708.0,
      "logps/rejected": -736.0,
      "loss": 0.5568,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.78125,
      "rewards/margins": 0.703125,
      "rewards/rejected": -4.5,
      "step": 660
    },
    {
      "epoch": 0.7012035583464155,
      "grad_norm": 1.9775753765391848,
      "learning_rate": 4.9754094494751364e-05,
      "logits/chosen": -6.125,
      "logits/rejected": -5.3125,
      "logps/chosen": -712.0,
      "logps/rejected": -764.0,
      "loss": 0.5442,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.734375,
      "rewards/margins": 0.69921875,
      "rewards/rejected": -4.4375,
      "step": 670
    },
    {
      "epoch": 0.7116692830978545,
      "grad_norm": 2.080545669185722,
      "learning_rate": 4.972786005862358e-05,
      "logits/chosen": -10.125,
      "logits/rejected": -9.5,
      "logps/chosen": -608.0,
      "logps/rejected": -632.0,
      "loss": 0.598,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.4375,
      "rewards/margins": 0.6171875,
      "rewards/rejected": -3.046875,
      "step": 680
    },
    {
      "epoch": 0.7221350078492935,
      "grad_norm": 1.7635314791728216,
      "learning_rate": 4.9700303859352e-05,
      "logits/chosen": -9.875,
      "logits/rejected": -9.3125,
      "logps/chosen": -536.0,
      "logps/rejected": -556.0,
      "loss": 0.537,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.171875,
      "rewards/margins": 0.74609375,
      "rewards/rejected": -2.921875,
      "step": 690
    },
    {
      "epoch": 0.7326007326007326,
      "grad_norm": 2.6926500474089856,
      "learning_rate": 4.967142736988127e-05,
      "logits/chosen": -9.25,
      "logits/rejected": -8.75,
      "logps/chosen": -568.0,
      "logps/rejected": -596.0,
      "loss": 0.5756,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -2.71875,
      "rewards/margins": 0.6015625,
      "rewards/rejected": -3.3125,
      "step": 700
    },
    {
      "epoch": 0.7326007326007326,
      "eval_logits/chosen": -9.375,
      "eval_logits/rejected": -8.8125,
      "eval_logps/chosen": -656.0,
      "eval_logps/rejected": -692.0,
      "eval_loss": 0.5859062671661377,
      "eval_rewards/accuracies": 0.701171875,
      "eval_rewards/chosen": -3.421875,
      "eval_rewards/margins": 0.61328125,
      "eval_rewards/rejected": -4.03125,
      "eval_runtime": 52.78,
      "eval_samples_per_second": 37.893,
      "eval_steps_per_second": 0.606,
      "step": 700
    },
    {
      "epoch": 0.7430664573521716,
      "grad_norm": 1.8338694406415879,
      "learning_rate": 4.964123213372865e-05,
      "logits/chosen": -8.625,
      "logits/rejected": -7.8125,
      "logps/chosen": -652.0,
      "logps/rejected": -672.0,
      "loss": 0.6116,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 0.50390625,
      "rewards/rejected": -4.0625,
      "step": 710
    },
    {
      "epoch": 0.7535321821036107,
      "grad_norm": 1.7715417270349543,
      "learning_rate": 4.96097197649016e-05,
      "logits/chosen": -9.25,
      "logits/rejected": -8.375,
      "logps/chosen": -624.0,
      "logps/rejected": -632.0,
      "loss": 0.5629,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -2.90625,
      "rewards/margins": 0.64453125,
      "rewards/rejected": -3.546875,
      "step": 720
    },
    {
      "epoch": 0.7639979068550498,
      "grad_norm": 2.09439081053431,
      "learning_rate": 4.9576891947811455e-05,
      "logits/chosen": -8.375,
      "logits/rejected": -7.75,
      "logps/chosen": -644.0,
      "logps/rejected": -648.0,
      "loss": 0.5916,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -3.328125,
      "rewards/margins": 0.486328125,
      "rewards/rejected": -3.8125,
      "step": 730
    },
    {
      "epoch": 0.7744636316064888,
      "grad_norm": 1.84005754928221,
      "learning_rate": 4.95427504371834e-05,
      "logits/chosen": -8.875,
      "logits/rejected": -8.25,
      "logps/chosen": -608.0,
      "logps/rejected": -644.0,
      "loss": 0.5471,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -3.171875,
      "rewards/margins": 0.65625,
      "rewards/rejected": -3.828125,
      "step": 740
    },
    {
      "epoch": 0.7849293563579278,
      "grad_norm": 1.6580778209495195,
      "learning_rate": 4.9507297057962645e-05,
      "logits/chosen": -10.375,
      "logits/rejected": -9.5625,
      "logps/chosen": -576.0,
      "logps/rejected": -640.0,
      "loss": 0.5362,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -2.96875,
      "rewards/margins": 0.71875,
      "rewards/rejected": -3.671875,
      "step": 750
    },
    {
      "epoch": 0.7953950811093669,
      "grad_norm": 1.834684453607076,
      "learning_rate": 4.947053370521696e-05,
      "logits/chosen": -9.0625,
      "logits/rejected": -7.8125,
      "logps/chosen": -560.0,
      "logps/rejected": -620.0,
      "loss": 0.5777,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.796875,
      "rewards/margins": 0.7109375,
      "rewards/rejected": -3.5,
      "step": 760
    },
    {
      "epoch": 0.8058608058608059,
      "grad_norm": 1.718132653359686,
      "learning_rate": 4.9432462344035294e-05,
      "logits/chosen": -9.25,
      "logits/rejected": -8.4375,
      "logps/chosen": -612.0,
      "logps/rejected": -680.0,
      "loss": 0.5631,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.65625,
      "rewards/margins": 0.51171875,
      "rewards/rejected": -3.171875,
      "step": 770
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 1.9148197442567063,
      "learning_rate": 4.939308500942277e-05,
      "logits/chosen": -5.9375,
      "logits/rejected": -4.75,
      "logps/chosen": -692.0,
      "logps/rejected": -696.0,
      "loss": 0.5711,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 0.68359375,
      "rewards/rejected": -4.03125,
      "step": 780
    },
    {
      "epoch": 0.826792255363684,
      "grad_norm": 1.9380251783145208,
      "learning_rate": 4.9352403806191926e-05,
      "logits/chosen": -3.96875,
      "logits/rejected": -2.625,
      "logps/chosen": -688.0,
      "logps/rejected": -700.0,
      "loss": 0.5952,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.609375,
      "rewards/margins": 0.625,
      "rewards/rejected": -4.25,
      "step": 790
    },
    {
      "epoch": 0.837257980115123,
      "grad_norm": 1.980206883814282,
      "learning_rate": 4.931042090885017e-05,
      "logits/chosen": -6.9375,
      "logits/rejected": -5.90625,
      "logps/chosen": -596.0,
      "logps/rejected": -644.0,
      "loss": 0.5528,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 0.6484375,
      "rewards/rejected": -3.5,
      "step": 800
    },
    {
      "epoch": 0.837257980115123,
      "eval_logits/chosen": -8.3125,
      "eval_logits/rejected": -7.4375,
      "eval_logps/chosen": -604.0,
      "eval_logps/rejected": -636.0,
      "eval_loss": 0.5731562376022339,
      "eval_rewards/accuracies": 0.68359375,
      "eval_rewards/chosen": -2.890625,
      "eval_rewards/margins": 0.6015625,
      "eval_rewards/rejected": -3.5,
      "eval_runtime": 52.7984,
      "eval_samples_per_second": 37.88,
      "eval_steps_per_second": 0.606,
      "step": 800
    },
    {
      "epoch": 0.847723704866562,
      "grad_norm": 1.970105831115586,
      "learning_rate": 4.926713856148357e-05,
      "logits/chosen": -7.8125,
      "logits/rejected": -6.84375,
      "logps/chosen": -652.0,
      "logps/rejected": -652.0,
      "loss": 0.5817,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -3.1875,
      "rewards/margins": 0.56640625,
      "rewards/rejected": -3.75,
      "step": 810
    },
    {
      "epoch": 0.858189429618001,
      "grad_norm": 2.163038230104847,
      "learning_rate": 4.922255907763692e-05,
      "logits/chosen": -7.46875,
      "logits/rejected": -6.46875,
      "logps/chosen": -684.0,
      "logps/rejected": -720.0,
      "loss": 0.5878,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.3125,
      "rewards/margins": 0.703125,
      "rewards/rejected": -4.0,
      "step": 820
    },
    {
      "epoch": 0.8686551543694401,
      "grad_norm": 1.5500963032625696,
      "learning_rate": 4.917668484019002e-05,
      "logits/chosen": -8.125,
      "logits/rejected": -7.46875,
      "logps/chosen": -580.0,
      "logps/rejected": -620.0,
      "loss": 0.5732,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 0.6015625,
      "rewards/rejected": -3.421875,
      "step": 830
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 2.2367725914269956,
      "learning_rate": 4.9129518301230375e-05,
      "logits/chosen": -7.28125,
      "logits/rejected": -5.84375,
      "logps/chosen": -616.0,
      "logps/rejected": -668.0,
      "loss": 0.5445,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -2.9375,
      "rewards/margins": 0.8125,
      "rewards/rejected": -3.75,
      "step": 840
    },
    {
      "epoch": 0.8895866038723181,
      "grad_norm": 2.4245666537731267,
      "learning_rate": 4.908106198192207e-05,
      "logits/chosen": -5.28125,
      "logits/rejected": -4.15625,
      "logps/chosen": -692.0,
      "logps/rejected": -704.0,
      "loss": 0.5262,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 0.828125,
      "rewards/rejected": -4.5,
      "step": 850
    },
    {
      "epoch": 0.9000523286237572,
      "grad_norm": 1.8299461987606431,
      "learning_rate": 4.903131847237104e-05,
      "logits/chosen": -6.40625,
      "logits/rejected": -5.09375,
      "logps/chosen": -736.0,
      "logps/rejected": -760.0,
      "loss": 0.5486,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -4.03125,
      "rewards/margins": 0.734375,
      "rewards/rejected": -4.78125,
      "step": 860
    },
    {
      "epoch": 0.9105180533751962,
      "grad_norm": 1.776519837344144,
      "learning_rate": 4.8980290431486595e-05,
      "logits/chosen": -9.0,
      "logits/rejected": -7.875,
      "logps/chosen": -612.0,
      "logps/rejected": -632.0,
      "loss": 0.6109,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 0.6328125,
      "rewards/rejected": -3.5625,
      "step": 870
    },
    {
      "epoch": 0.9209837781266352,
      "grad_norm": 1.5665799264044165,
      "learning_rate": 4.892798058683933e-05,
      "logits/chosen": -7.5625,
      "logits/rejected": -6.71875,
      "logps/chosen": -636.0,
      "logps/rejected": -692.0,
      "loss": 0.5469,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.921875,
      "rewards/margins": 0.6640625,
      "rewards/rejected": -3.59375,
      "step": 880
    },
    {
      "epoch": 0.9314495028780743,
      "grad_norm": 1.6938125314327865,
      "learning_rate": 4.887439173451529e-05,
      "logits/chosen": -8.3125,
      "logits/rejected": -7.375,
      "logps/chosen": -628.0,
      "logps/rejected": -668.0,
      "loss": 0.5701,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 0.6640625,
      "rewards/rejected": -3.546875,
      "step": 890
    },
    {
      "epoch": 0.9419152276295133,
      "grad_norm": 1.9001080952998146,
      "learning_rate": 4.8819526738966546e-05,
      "logits/chosen": -8.625,
      "logits/rejected": -7.9375,
      "logps/chosen": -616.0,
      "logps/rejected": -668.0,
      "loss": 0.5753,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.90625,
      "rewards/margins": 0.59765625,
      "rewards/rejected": -3.5,
      "step": 900
    },
    {
      "epoch": 0.9419152276295133,
      "eval_logits/chosen": -7.90625,
      "eval_logits/rejected": -7.0,
      "eval_logps/chosen": -620.0,
      "eval_logps/rejected": -660.0,
      "eval_loss": 0.5692968964576721,
      "eval_rewards/accuracies": 0.716796875,
      "eval_rewards/chosen": -3.046875,
      "eval_rewards/margins": 0.6796875,
      "eval_rewards/rejected": -3.734375,
      "eval_runtime": 52.8034,
      "eval_samples_per_second": 37.876,
      "eval_steps_per_second": 0.606,
      "step": 900
    },
    {
      "epoch": 0.9523809523809523,
      "grad_norm": 1.8582669591039784,
      "learning_rate": 4.876338853285809e-05,
      "logits/chosen": -8.375,
      "logits/rejected": -7.59375,
      "logps/chosen": -532.0,
      "logps/rejected": -572.0,
      "loss": 0.593,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -2.75,
      "rewards/margins": 0.57421875,
      "rewards/rejected": -3.328125,
      "step": 910
    },
    {
      "epoch": 0.9628466771323915,
      "grad_norm": 1.927805055357823,
      "learning_rate": 4.870598011691101e-05,
      "logits/chosen": -9.5,
      "logits/rejected": -9.0,
      "logps/chosen": -628.0,
      "logps/rejected": -636.0,
      "loss": 0.5746,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -3.046875,
      "rewards/margins": 0.55078125,
      "rewards/rejected": -3.59375,
      "step": 920
    },
    {
      "epoch": 0.9733124018838305,
      "grad_norm": 1.5760630448625366,
      "learning_rate": 4.864730455974218e-05,
      "logits/chosen": -7.34375,
      "logits/rejected": -6.4375,
      "logps/chosen": -760.0,
      "logps/rejected": -756.0,
      "loss": 0.5694,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -4.0,
      "rewards/margins": 0.640625,
      "rewards/rejected": -4.65625,
      "step": 930
    },
    {
      "epoch": 0.9837781266352695,
      "grad_norm": 1.7670941807840217,
      "learning_rate": 4.858736499770018e-05,
      "logits/chosen": -8.4375,
      "logits/rejected": -7.59375,
      "logps/chosen": -652.0,
      "logps/rejected": -692.0,
      "loss": 0.5349,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 0.63671875,
      "rewards/rejected": -4.125,
      "step": 940
    },
    {
      "epoch": 0.9942438513867086,
      "grad_norm": 1.8451915529649179,
      "learning_rate": 4.852616463469768e-05,
      "logits/chosen": -8.125,
      "logits/rejected": -7.4375,
      "logps/chosen": -708.0,
      "logps/rejected": -744.0,
      "loss": 0.5689,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 0.625,
      "rewards/rejected": -4.21875,
      "step": 950
    },
    {
      "epoch": 1.0047095761381475,
      "grad_norm": 1.6173131691447116,
      "learning_rate": 4.846370674204016e-05,
      "logits/chosen": -8.5625,
      "logits/rejected": -7.90625,
      "logps/chosen": -644.0,
      "logps/rejected": -692.0,
      "loss": 0.47,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 0.90625,
      "rewards/rejected": -4.1875,
      "step": 960
    },
    {
      "epoch": 1.0151753008895865,
      "grad_norm": 1.865923958174982,
      "learning_rate": 4.839999465825107e-05,
      "logits/chosen": -6.5,
      "logits/rejected": -5.125,
      "logps/chosen": -776.0,
      "logps/rejected": -964.0,
      "loss": 0.2891,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.40625,
      "rewards/rejected": -6.59375,
      "step": 970
    },
    {
      "epoch": 1.0256410256410255,
      "grad_norm": 1.9783894181617947,
      "learning_rate": 4.8335031788893346e-05,
      "logits/chosen": -5.28125,
      "logits/rejected": -3.0,
      "logps/chosen": -752.0,
      "logps/rejected": -944.0,
      "loss": 0.2655,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.5,
      "rewards/margins": 2.421875,
      "rewards/rejected": -6.9375,
      "step": 980
    },
    {
      "epoch": 1.0361067503924646,
      "grad_norm": 1.6618530076050113,
      "learning_rate": 4.826882160638744e-05,
      "logits/chosen": -6.5625,
      "logits/rejected": -4.15625,
      "logps/chosen": -728.0,
      "logps/rejected": -956.0,
      "loss": 0.2357,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.90625,
      "rewards/margins": 2.84375,
      "rewards/rejected": -6.75,
      "step": 990
    },
    {
      "epoch": 1.0465724751439036,
      "grad_norm": 1.6852553538671544,
      "learning_rate": 4.820136764982564e-05,
      "logits/chosen": -5.8125,
      "logits/rejected": -3.984375,
      "logps/chosen": -704.0,
      "logps/rejected": -896.0,
      "loss": 0.2632,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.546875,
      "rewards/margins": 2.375,
      "rewards/rejected": -5.9375,
      "step": 1000
    },
    {
      "epoch": 1.0465724751439036,
      "eval_logits/chosen": -4.25,
      "eval_logits/rejected": -2.875,
      "eval_logps/chosen": -732.0,
      "eval_logps/rejected": -808.0,
      "eval_loss": 0.5880781412124634,
      "eval_rewards/accuracies": 0.71484375,
      "eval_rewards/chosen": -4.1875,
      "eval_rewards/margins": 1.03125,
      "eval_rewards/rejected": -5.21875,
      "eval_runtime": 52.7927,
      "eval_samples_per_second": 37.884,
      "eval_steps_per_second": 0.606,
      "step": 1000
    },
    {
      "epoch": 1.0570381998953426,
      "grad_norm": 2.6176318352128414,
      "learning_rate": 4.813267352478292e-05,
      "logits/chosen": -4.40625,
      "logits/rejected": -2.453125,
      "logps/chosen": -660.0,
      "logps/rejected": -836.0,
      "loss": 0.2373,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.4375,
      "rewards/margins": 2.375,
      "rewards/rejected": -5.8125,
      "step": 1010
    },
    {
      "epoch": 1.0675039246467817,
      "grad_norm": 4.188123637402671,
      "learning_rate": 4.8062742903124246e-05,
      "logits/chosen": -3.53125,
      "logits/rejected": -2.0625,
      "logps/chosen": -760.0,
      "logps/rejected": -996.0,
      "loss": 0.2263,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.125,
      "rewards/margins": 2.6875,
      "rewards/rejected": -6.8125,
      "step": 1020
    },
    {
      "epoch": 1.077969649398221,
      "grad_norm": 1.6827915651039376,
      "learning_rate": 4.799157952280826e-05,
      "logits/chosen": -4.8125,
      "logits/rejected": -2.984375,
      "logps/chosen": -700.0,
      "logps/rejected": -904.0,
      "loss": 0.2376,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.765625,
      "rewards/margins": 2.453125,
      "rewards/rejected": -6.21875,
      "step": 1030
    },
    {
      "epoch": 1.08843537414966,
      "grad_norm": 1.6536012233098174,
      "learning_rate": 4.7919187187687496e-05,
      "logits/chosen": -3.90625,
      "logits/rejected": -2.03125,
      "logps/chosen": -672.0,
      "logps/rejected": -860.0,
      "loss": 0.2642,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 2.28125,
      "rewards/rejected": -6.0,
      "step": 1040
    },
    {
      "epoch": 1.098901098901099,
      "grad_norm": 2.774611600374004,
      "learning_rate": 4.784556976730507e-05,
      "logits/chosen": -6.0625,
      "logits/rejected": -4.4375,
      "logps/chosen": -632.0,
      "logps/rejected": -868.0,
      "loss": 0.1976,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 2.703125,
      "rewards/rejected": -6.03125,
      "step": 1050
    },
    {
      "epoch": 1.109366823652538,
      "grad_norm": 2.4394840573251284,
      "learning_rate": 4.777073119668782e-05,
      "logits/chosen": -4.46875,
      "logits/rejected": -2.1875,
      "logps/chosen": -760.0,
      "logps/rejected": -968.0,
      "loss": 0.236,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.40625,
      "rewards/margins": 2.53125,
      "rewards/rejected": -6.9375,
      "step": 1060
    },
    {
      "epoch": 1.119832548403977,
      "grad_norm": 3.093080712317507,
      "learning_rate": 4.769467547613598e-05,
      "logits/chosen": -5.21875,
      "logits/rejected": -2.53125,
      "logps/chosen": -752.0,
      "logps/rejected": -972.0,
      "loss": 0.2397,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.28125,
      "rewards/margins": 2.6875,
      "rewards/rejected": -7.0,
      "step": 1070
    },
    {
      "epoch": 1.130298273155416,
      "grad_norm": 3.8414209437581888,
      "learning_rate": 4.761740667100934e-05,
      "logits/chosen": -6.1875,
      "logits/rejected": -4.53125,
      "logps/chosen": -708.0,
      "logps/rejected": -980.0,
      "loss": 0.2282,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.09375,
      "rewards/margins": 2.890625,
      "rewards/rejected": -7.0,
      "step": 1080
    },
    {
      "epoch": 1.1407639979068551,
      "grad_norm": 1.3714400548829968,
      "learning_rate": 4.7538928911509986e-05,
      "logits/chosen": -6.1875,
      "logits/rejected": -3.953125,
      "logps/chosen": -748.0,
      "logps/rejected": -1024.0,
      "loss": 0.2141,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.28125,
      "rewards/margins": 3.03125,
      "rewards/rejected": -7.3125,
      "step": 1090
    },
    {
      "epoch": 1.1512297226582942,
      "grad_norm": 2.8001835210684174,
      "learning_rate": 4.745924639246148e-05,
      "logits/chosen": -5.875,
      "logits/rejected": -4.125,
      "logps/chosen": -720.0,
      "logps/rejected": -920.0,
      "loss": 0.2283,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.90625,
      "rewards/margins": 2.40625,
      "rewards/rejected": -6.3125,
      "step": 1100
    },
    {
      "epoch": 1.1512297226582942,
      "eval_logits/chosen": -7.0625,
      "eval_logits/rejected": -5.375,
      "eval_logps/chosen": -768.0,
      "eval_logps/rejected": -840.0,
      "eval_loss": 0.6142030954360962,
      "eval_rewards/accuracies": 0.712890625,
      "eval_rewards/chosen": -4.53125,
      "eval_rewards/margins": 0.99609375,
      "eval_rewards/rejected": -5.53125,
      "eval_runtime": 52.7843,
      "eval_samples_per_second": 37.89,
      "eval_steps_per_second": 0.606,
      "step": 1100
    },
    {
      "epoch": 1.1616954474097332,
      "grad_norm": 1.8190177892297257,
      "learning_rate": 4.737836337308465e-05,
      "logits/chosen": -6.375,
      "logits/rejected": -3.796875,
      "logps/chosen": -768.0,
      "logps/rejected": -1016.0,
      "loss": 0.1826,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.5625,
      "rewards/margins": 2.875,
      "rewards/rejected": -7.4375,
      "step": 1110
    },
    {
      "epoch": 1.1721611721611722,
      "grad_norm": 2.4839194991731746,
      "learning_rate": 4.729628417676997e-05,
      "logits/chosen": -4.0625,
      "logits/rejected": -2.1875,
      "logps/chosen": -816.0,
      "logps/rejected": -1072.0,
      "loss": 0.2546,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.9375,
      "rewards/margins": 2.84375,
      "rewards/rejected": -7.78125,
      "step": 1120
    },
    {
      "epoch": 1.1826268969126112,
      "grad_norm": 1.7415070251510811,
      "learning_rate": 4.7213013190846375e-05,
      "logits/chosen": -6.1875,
      "logits/rejected": -3.921875,
      "logps/chosen": -756.0,
      "logps/rejected": -996.0,
      "loss": 0.192,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.25,
      "rewards/margins": 2.671875,
      "rewards/rejected": -6.90625,
      "step": 1130
    },
    {
      "epoch": 1.1930926216640503,
      "grad_norm": 2.6883474669097196,
      "learning_rate": 4.712855486634684e-05,
      "logits/chosen": -6.15625,
      "logits/rejected": -4.3125,
      "logps/chosen": -832.0,
      "logps/rejected": -1096.0,
      "loss": 0.2138,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.78125,
      "rewards/margins": 3.078125,
      "rewards/rejected": -7.84375,
      "step": 1140
    },
    {
      "epoch": 1.2035583464154893,
      "grad_norm": 3.3034116848079753,
      "learning_rate": 4.7042913717770385e-05,
      "logits/chosen": -5.4375,
      "logits/rejected": -3.953125,
      "logps/chosen": -800.0,
      "logps/rejected": -1072.0,
      "loss": 0.203,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.84375,
      "rewards/margins": 2.8125,
      "rewards/rejected": -7.65625,
      "step": 1150
    },
    {
      "epoch": 1.2140240711669283,
      "grad_norm": 3.083004913157005,
      "learning_rate": 4.695609432284085e-05,
      "logits/chosen": -6.0625,
      "logits/rejected": -3.34375,
      "logps/chosen": -804.0,
      "logps/rejected": -1032.0,
      "loss": 0.2456,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.90625,
      "rewards/margins": 2.75,
      "rewards/rejected": -7.65625,
      "step": 1160
    },
    {
      "epoch": 1.2244897959183674,
      "grad_norm": 2.3706996918578445,
      "learning_rate": 4.68681013222621e-05,
      "logits/chosen": -5.59375,
      "logits/rejected": -3.65625,
      "logps/chosen": -820.0,
      "logps/rejected": -1056.0,
      "loss": 0.2102,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.09375,
      "rewards/margins": 2.78125,
      "rewards/rejected": -7.875,
      "step": 1170
    },
    {
      "epoch": 1.2349555206698064,
      "grad_norm": 3.001358387229265,
      "learning_rate": 4.677893941947006e-05,
      "logits/chosen": -6.53125,
      "logits/rejected": -4.75,
      "logps/chosen": -736.0,
      "logps/rejected": -980.0,
      "loss": 0.2553,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.15625,
      "rewards/margins": 2.859375,
      "rewards/rejected": -7.03125,
      "step": 1180
    },
    {
      "epoch": 1.2454212454212454,
      "grad_norm": 2.714225581231745,
      "learning_rate": 4.668861338038126e-05,
      "logits/chosen": -5.34375,
      "logits/rejected": -3.390625,
      "logps/chosen": -728.0,
      "logps/rejected": -928.0,
      "loss": 0.2101,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.953125,
      "rewards/margins": 2.5,
      "rewards/rejected": -6.46875,
      "step": 1190
    },
    {
      "epoch": 1.2558869701726845,
      "grad_norm": 2.7020928838849194,
      "learning_rate": 4.6597128033138095e-05,
      "logits/chosen": -3.984375,
      "logits/rejected": -1.984375,
      "logps/chosen": -740.0,
      "logps/rejected": -960.0,
      "loss": 0.2202,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.65625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -6.375,
      "step": 1200
    },
    {
      "epoch": 1.2558869701726845,
      "eval_logits/chosen": -3.375,
      "eval_logits/rejected": -1.875,
      "eval_logps/chosen": -724.0,
      "eval_logps/rejected": -804.0,
      "eval_loss": 0.5942812561988831,
      "eval_rewards/accuracies": 0.708984375,
      "eval_rewards/chosen": -4.09375,
      "eval_rewards/margins": 1.078125,
      "eval_rewards/rejected": -5.1875,
      "eval_runtime": 52.81,
      "eval_samples_per_second": 37.872,
      "eval_steps_per_second": 0.606,
      "step": 1200
    },
    {
      "epoch": 1.2663526949241235,
      "grad_norm": 2.3938000016341037,
      "learning_rate": 4.650448826785073e-05,
      "logits/chosen": -4.25,
      "logits/rejected": -1.9765625,
      "logps/chosen": -776.0,
      "logps/rejected": -944.0,
      "loss": 0.238,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.875,
      "rewards/margins": 2.59375,
      "rewards/rejected": -6.46875,
      "step": 1210
    },
    {
      "epoch": 1.2768184196755625,
      "grad_norm": 2.4095410418117633,
      "learning_rate": 4.641069903633575e-05,
      "logits/chosen": -3.921875,
      "logits/rejected": -2.390625,
      "logps/chosen": -756.0,
      "logps/rejected": -956.0,
      "loss": 0.2538,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.34375,
      "rewards/margins": 2.5,
      "rewards/rejected": -6.8125,
      "step": 1220
    },
    {
      "epoch": 1.2872841444270016,
      "grad_norm": 2.71326099150034,
      "learning_rate": 4.631576535185146e-05,
      "logits/chosen": -5.15625,
      "logits/rejected": -3.375,
      "logps/chosen": -648.0,
      "logps/rejected": -864.0,
      "loss": 0.2384,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.5,
      "rewards/margins": 2.609375,
      "rewards/rejected": -6.09375,
      "step": 1230
    },
    {
      "epoch": 1.2977498691784406,
      "grad_norm": 2.743366901868589,
      "learning_rate": 4.621969228882987e-05,
      "logits/chosen": -6.25,
      "logits/rejected": -4.34375,
      "logps/chosen": -684.0,
      "logps/rejected": -936.0,
      "loss": 0.2121,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.8125,
      "rewards/margins": 2.796875,
      "rewards/rejected": -6.625,
      "step": 1240
    },
    {
      "epoch": 1.3082155939298796,
      "grad_norm": 3.3400284830383797,
      "learning_rate": 4.612248498260554e-05,
      "logits/chosen": -6.6875,
      "logits/rejected": -4.25,
      "logps/chosen": -728.0,
      "logps/rejected": -944.0,
      "loss": 0.2304,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.890625,
      "rewards/margins": 2.828125,
      "rewards/rejected": -6.71875,
      "step": 1250
    },
    {
      "epoch": 1.3186813186813187,
      "grad_norm": 2.6309946499126577,
      "learning_rate": 4.602414862914101e-05,
      "logits/chosen": -5.21875,
      "logits/rejected": -3.703125,
      "logps/chosen": -720.0,
      "logps/rejected": -960.0,
      "loss": 0.2174,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.59375,
      "rewards/rejected": -6.78125,
      "step": 1260
    },
    {
      "epoch": 1.3291470434327577,
      "grad_norm": 2.2785673557762522,
      "learning_rate": 4.5924688484749087e-05,
      "logits/chosen": -4.40625,
      "logits/rejected": -2.375,
      "logps/chosen": -764.0,
      "logps/rejected": -992.0,
      "loss": 0.2348,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.53125,
      "rewards/margins": 2.609375,
      "rewards/rejected": -7.125,
      "step": 1270
    },
    {
      "epoch": 1.3396127681841967,
      "grad_norm": 2.676901490717523,
      "learning_rate": 4.582410986581191e-05,
      "logits/chosen": -2.6875,
      "logits/rejected": -0.134765625,
      "logps/chosen": -812.0,
      "logps/rejected": -1064.0,
      "loss": 0.2292,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.9375,
      "step": 1280
    },
    {
      "epoch": 1.3500784929356358,
      "grad_norm": 2.748893098182492,
      "learning_rate": 4.572241814849672e-05,
      "logits/chosen": -2.4375,
      "logits/rejected": -0.58984375,
      "logps/chosen": -764.0,
      "logps/rejected": -1004.0,
      "loss": 0.2677,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.34375,
      "rewards/margins": 2.84375,
      "rewards/rejected": -7.1875,
      "step": 1290
    },
    {
      "epoch": 1.3605442176870748,
      "grad_norm": 2.5611387716985607,
      "learning_rate": 4.561961876846853e-05,
      "logits/chosen": -3.96875,
      "logits/rejected": -2.5,
      "logps/chosen": -684.0,
      "logps/rejected": -912.0,
      "loss": 0.2472,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 2.359375,
      "rewards/rejected": -6.0625,
      "step": 1300
    },
    {
      "epoch": 1.3605442176870748,
      "eval_logits/chosen": -3.6875,
      "eval_logits/rejected": -2.21875,
      "eval_logps/chosen": -760.0,
      "eval_logps/rejected": -828.0,
      "eval_loss": 0.5994687676429749,
      "eval_rewards/accuracies": 0.716796875,
      "eval_rewards/chosen": -4.4375,
      "eval_rewards/margins": 0.984375,
      "eval_rewards/rejected": -5.40625,
      "eval_runtime": 52.7983,
      "eval_samples_per_second": 37.88,
      "eval_steps_per_second": 0.606,
      "step": 1300
    },
    {
      "epoch": 1.3710099424385138,
      "grad_norm": 1.6282540391036573,
      "learning_rate": 4.55157172205996e-05,
      "logits/chosen": -4.25,
      "logits/rejected": -1.953125,
      "logps/chosen": -756.0,
      "logps/rejected": -940.0,
      "loss": 0.2387,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.25,
      "rewards/margins": 2.390625,
      "rewards/rejected": -6.65625,
      "step": 1310
    },
    {
      "epoch": 1.3814756671899528,
      "grad_norm": 3.4806185138527828,
      "learning_rate": 4.5410719058675654e-05,
      "logits/chosen": -3.203125,
      "logits/rejected": -1.8828125,
      "logps/chosen": -744.0,
      "logps/rejected": -972.0,
      "loss": 0.2337,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.5625,
      "rewards/margins": 2.5,
      "rewards/rejected": -7.0625,
      "step": 1320
    },
    {
      "epoch": 1.3919413919413919,
      "grad_norm": 2.7953097923118806,
      "learning_rate": 4.5304629895099085e-05,
      "logits/chosen": -3.828125,
      "logits/rejected": -2.78125,
      "logps/chosen": -760.0,
      "logps/rejected": -1040.0,
      "loss": 0.2346,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.46875,
      "rewards/margins": 2.6875,
      "rewards/rejected": -7.15625,
      "step": 1330
    },
    {
      "epoch": 1.402407116692831,
      "grad_norm": 2.5716033588233125,
      "learning_rate": 4.51974554005889e-05,
      "logits/chosen": -3.9375,
      "logits/rejected": -1.5703125,
      "logps/chosen": -804.0,
      "logps/rejected": -1032.0,
      "loss": 0.2277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.6875,
      "rewards/margins": 2.8125,
      "rewards/rejected": -7.5,
      "step": 1340
    },
    {
      "epoch": 1.41287284144427,
      "grad_norm": 2.672897305019244,
      "learning_rate": 4.508920130387764e-05,
      "logits/chosen": -3.953125,
      "logits/rejected": -1.7421875,
      "logps/chosen": -724.0,
      "logps/rejected": -1004.0,
      "loss": 0.2336,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.09375,
      "rewards/margins": 3.078125,
      "rewards/rejected": -7.1875,
      "step": 1350
    },
    {
      "epoch": 1.423338566195709,
      "grad_norm": 2.185839165890125,
      "learning_rate": 4.497987339140519e-05,
      "logits/chosen": -5.03125,
      "logits/rejected": -3.140625,
      "logps/chosen": -700.0,
      "logps/rejected": -920.0,
      "loss": 0.2334,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.796875,
      "rewards/margins": 2.6875,
      "rewards/rejected": -6.46875,
      "step": 1360
    },
    {
      "epoch": 1.433804290947148,
      "grad_norm": 3.9172194922718746,
      "learning_rate": 4.4869477507009424e-05,
      "logits/chosen": -4.4375,
      "logits/rejected": -2.859375,
      "logps/chosen": -768.0,
      "logps/rejected": -988.0,
      "loss": 0.2671,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.625,
      "rewards/margins": 2.546875,
      "rewards/rejected": -7.1875,
      "step": 1370
    },
    {
      "epoch": 1.4442700156985873,
      "grad_norm": 1.5309369238147958,
      "learning_rate": 4.475801955161386e-05,
      "logits/chosen": -4.96875,
      "logits/rejected": -2.859375,
      "logps/chosen": -796.0,
      "logps/rejected": -1024.0,
      "loss": 0.2203,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.78125,
      "rewards/margins": 2.71875,
      "rewards/rejected": -7.5,
      "step": 1380
    },
    {
      "epoch": 1.454735740450026,
      "grad_norm": 2.400254274178371,
      "learning_rate": 4.464550548291227e-05,
      "logits/chosen": -6.1875,
      "logits/rejected": -3.9375,
      "logps/chosen": -760.0,
      "logps/rejected": -976.0,
      "loss": 0.2565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.375,
      "rewards/margins": 2.734375,
      "rewards/rejected": -7.09375,
      "step": 1390
    },
    {
      "epoch": 1.4652014652014653,
      "grad_norm": 3.2947707252238945,
      "learning_rate": 4.453194131505018e-05,
      "logits/chosen": -6.1875,
      "logits/rejected": -5.0,
      "logps/chosen": -816.0,
      "logps/rejected": -1056.0,
      "loss": 0.2406,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.75,
      "rewards/margins": 2.5,
      "rewards/rejected": -7.25,
      "step": 1400
    },
    {
      "epoch": 1.4652014652014653,
      "eval_logits/chosen": -5.28125,
      "eval_logits/rejected": -3.875,
      "eval_logps/chosen": -836.0,
      "eval_logps/rejected": -908.0,
      "eval_loss": 0.5971249938011169,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -5.21875,
      "eval_rewards/margins": 1.015625,
      "eval_rewards/rejected": -6.21875,
      "eval_runtime": 52.8282,
      "eval_samples_per_second": 37.859,
      "eval_steps_per_second": 0.606,
      "step": 1400
    },
    {
      "epoch": 1.4756671899529041,
      "grad_norm": 2.504445176515428,
      "learning_rate": 4.4417333118303434e-05,
      "logits/chosen": -4.9375,
      "logits/rejected": -2.90625,
      "logps/chosen": -764.0,
      "logps/rejected": -1016.0,
      "loss": 0.2169,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.5625,
      "rewards/margins": 2.734375,
      "rewards/rejected": -7.28125,
      "step": 1410
    },
    {
      "epoch": 1.4861329147043434,
      "grad_norm": 1.890385623018933,
      "learning_rate": 4.430168701875371e-05,
      "logits/chosen": -1.765625,
      "logits/rejected": -0.3359375,
      "logps/chosen": -844.0,
      "logps/rejected": -1064.0,
      "loss": 0.227,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.03125,
      "rewards/margins": 2.671875,
      "rewards/rejected": -7.71875,
      "step": 1420
    },
    {
      "epoch": 1.4965986394557822,
      "grad_norm": 3.1961582028367768,
      "learning_rate": 4.418500919796107e-05,
      "logits/chosen": -2.5625,
      "logits/rejected": -0.37890625,
      "logps/chosen": -828.0,
      "logps/rejected": -1128.0,
      "loss": 0.2094,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.328125,
      "rewards/rejected": -8.25,
      "step": 1430
    },
    {
      "epoch": 1.5070643642072215,
      "grad_norm": 1.7802682782877906,
      "learning_rate": 4.4067305892633526e-05,
      "logits/chosen": -3.1875,
      "logits/rejected": -1.2734375,
      "logps/chosen": -844.0,
      "logps/rejected": -1120.0,
      "loss": 0.2196,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.171875,
      "rewards/rejected": -8.1875,
      "step": 1440
    },
    {
      "epoch": 1.5175300889586603,
      "grad_norm": 2.835575157287789,
      "learning_rate": 4.3948583394293716e-05,
      "logits/chosen": -3.671875,
      "logits/rejected": -2.046875,
      "logps/chosen": -924.0,
      "logps/rejected": -1168.0,
      "loss": 0.2255,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.5625,
      "rewards/margins": 2.9375,
      "rewards/rejected": -8.5,
      "step": 1450
    },
    {
      "epoch": 1.5279958137100995,
      "grad_norm": 2.68427303811402,
      "learning_rate": 4.382884804894255e-05,
      "logits/chosen": -4.4375,
      "logits/rejected": -2.5625,
      "logps/chosen": -820.0,
      "logps/rejected": -1120.0,
      "loss": 0.1839,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.9375,
      "rewards/margins": 3.078125,
      "rewards/rejected": -8.0,
      "step": 1460
    },
    {
      "epoch": 1.5384615384615383,
      "grad_norm": 2.9253750364643425,
      "learning_rate": 4.3708106256720025e-05,
      "logits/chosen": -5.59375,
      "logits/rejected": -4.03125,
      "logps/chosen": -772.0,
      "logps/rejected": -1048.0,
      "loss": 0.2311,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.15625,
      "rewards/margins": 2.90625,
      "rewards/rejected": -7.09375,
      "step": 1470
    },
    {
      "epoch": 1.5489272632129776,
      "grad_norm": 1.9394845104409457,
      "learning_rate": 4.358636447156316e-05,
      "logits/chosen": -4.40625,
      "logits/rejected": -2.3125,
      "logps/chosen": -744.0,
      "logps/rejected": -928.0,
      "loss": 0.2532,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -4.46875,
      "rewards/margins": 2.46875,
      "rewards/rejected": -6.9375,
      "step": 1480
    },
    {
      "epoch": 1.5593929879644164,
      "grad_norm": 2.018780263579091,
      "learning_rate": 4.346362920086094e-05,
      "logits/chosen": -5.09375,
      "logits/rejected": -2.953125,
      "logps/chosen": -828.0,
      "logps/rejected": -1016.0,
      "loss": 0.2116,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -4.90625,
      "rewards/margins": 2.34375,
      "rewards/rejected": -7.25,
      "step": 1490
    },
    {
      "epoch": 1.5698587127158556,
      "grad_norm": 1.9518425840610047,
      "learning_rate": 4.333990700510656e-05,
      "logits/chosen": -6.125,
      "logits/rejected": -4.15625,
      "logps/chosen": -832.0,
      "logps/rejected": -1080.0,
      "loss": 0.2059,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.9375,
      "rewards/margins": 2.703125,
      "rewards/rejected": -7.65625,
      "step": 1500
    },
    {
      "epoch": 1.5698587127158556,
      "eval_logits/chosen": -5.78125,
      "eval_logits/rejected": -4.21875,
      "eval_logps/chosen": -848.0,
      "eval_logps/rejected": -940.0,
      "eval_loss": 0.6052031517028809,
      "eval_rewards/accuracies": 0.71484375,
      "eval_rewards/chosen": -5.34375,
      "eval_rewards/margins": 1.1953125,
      "eval_rewards/rejected": -6.53125,
      "eval_runtime": 52.836,
      "eval_samples_per_second": 37.853,
      "eval_steps_per_second": 0.606,
      "step": 1500
    },
    {
      "epoch": 1.5803244374672945,
      "grad_norm": 2.1970202585834695,
      "learning_rate": 4.321520449754669e-05,
      "logits/chosen": -6.15625,
      "logits/rejected": -4.34375,
      "logps/chosen": -820.0,
      "logps/rejected": -1072.0,
      "loss": 0.2231,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.71875,
      "rewards/margins": 2.890625,
      "rewards/rejected": -7.625,
      "step": 1510
    },
    {
      "epoch": 1.5907901622187337,
      "grad_norm": 2.7541131093791233,
      "learning_rate": 4.308952834382802e-05,
      "logits/chosen": -7.75,
      "logits/rejected": -5.75,
      "logps/chosen": -816.0,
      "logps/rejected": -1048.0,
      "loss": 0.2373,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.8125,
      "rewards/margins": 2.640625,
      "rewards/rejected": -7.4375,
      "step": 1520
    },
    {
      "epoch": 1.6012558869701727,
      "grad_norm": 2.1173531392884413,
      "learning_rate": 4.296288526164094e-05,
      "logits/chosen": -7.1875,
      "logits/rejected": -4.5625,
      "logps/chosen": -836.0,
      "logps/rejected": -1056.0,
      "loss": 0.2056,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.0625,
      "rewards/margins": 2.9375,
      "rewards/rejected": -8.0,
      "step": 1530
    },
    {
      "epoch": 1.6117216117216118,
      "grad_norm": 1.871611457195126,
      "learning_rate": 4.283528202036049e-05,
      "logits/chosen": -8.0625,
      "logits/rejected": -5.78125,
      "logps/chosen": -808.0,
      "logps/rejected": -1016.0,
      "loss": 0.223,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.59375,
      "rewards/margins": 2.640625,
      "rewards/rejected": -7.25,
      "step": 1540
    },
    {
      "epoch": 1.6221873364730508,
      "grad_norm": 1.7973566338479199,
      "learning_rate": 4.270672544068448e-05,
      "logits/chosen": -8.3125,
      "logits/rejected": -6.1875,
      "logps/chosen": -816.0,
      "logps/rejected": -1080.0,
      "loss": 0.2537,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.65625,
      "rewards/margins": 2.890625,
      "rewards/rejected": -7.5625,
      "step": 1550
    },
    {
      "epoch": 1.6326530612244898,
      "grad_norm": 2.0569039776175493,
      "learning_rate": 4.257722239426897e-05,
      "logits/chosen": -7.75,
      "logits/rejected": -5.34375,
      "logps/chosen": -784.0,
      "logps/rejected": -1024.0,
      "loss": 0.2426,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.4375,
      "rewards/margins": 2.90625,
      "rewards/rejected": -7.375,
      "step": 1560
    },
    {
      "epoch": 1.6431187859759289,
      "grad_norm": 2.0178296101250197,
      "learning_rate": 4.2446779803360895e-05,
      "logits/chosen": -6.1875,
      "logits/rejected": -4.03125,
      "logps/chosen": -772.0,
      "logps/rejected": -996.0,
      "loss": 0.2115,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.6875,
      "rewards/margins": 2.6875,
      "rewards/rejected": -7.375,
      "step": 1570
    },
    {
      "epoch": 1.653584510727368,
      "grad_norm": 1.79335431596216,
      "learning_rate": 4.23154046404281e-05,
      "logits/chosen": -6.0625,
      "logits/rejected": -3.625,
      "logps/chosen": -848.0,
      "logps/rejected": -1080.0,
      "loss": 0.1949,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.1875,
      "rewards/margins": 2.890625,
      "rewards/rejected": -8.0625,
      "step": 1580
    },
    {
      "epoch": 1.664050235478807,
      "grad_norm": 2.290555032802585,
      "learning_rate": 4.218310392778662e-05,
      "logits/chosen": -5.125,
      "logits/rejected": -2.296875,
      "logps/chosen": -880.0,
      "logps/rejected": -1144.0,
      "loss": 0.229,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.015625,
      "rewards/rejected": -8.5625,
      "step": 1590
    },
    {
      "epoch": 1.674515960230246,
      "grad_norm": 2.115801499090191,
      "learning_rate": 4.204988473722535e-05,
      "logits/chosen": -5.625,
      "logits/rejected": -3.40625,
      "logps/chosen": -832.0,
      "logps/rejected": -1072.0,
      "loss": 0.2305,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.9375,
      "rewards/margins": 2.921875,
      "rewards/rejected": -7.84375,
      "step": 1600
    },
    {
      "epoch": 1.674515960230246,
      "eval_logits/chosen": -6.6875,
      "eval_logits/rejected": -5.15625,
      "eval_logps/chosen": -800.0,
      "eval_logps/rejected": -876.0,
      "eval_loss": 0.6067500114440918,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -4.875,
      "eval_rewards/margins": 1.0390625,
      "eval_rewards/rejected": -5.90625,
      "eval_runtime": 52.8557,
      "eval_samples_per_second": 37.839,
      "eval_steps_per_second": 0.605,
      "step": 1600
    },
    {
      "epoch": 1.684981684981685,
      "grad_norm": 1.8649325471458336,
      "learning_rate": 4.1915754189628e-05,
      "logits/chosen": -8.0,
      "logits/rejected": -5.90625,
      "logps/chosen": -796.0,
      "logps/rejected": -1040.0,
      "loss": 0.2418,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.375,
      "rewards/margins": 2.78125,
      "rewards/rejected": -7.15625,
      "step": 1610
    },
    {
      "epoch": 1.695447409733124,
      "grad_norm": 2.3340385989698715,
      "learning_rate": 4.1780719454592496e-05,
      "logits/chosen": -6.9375,
      "logits/rejected": -4.6875,
      "logps/chosen": -744.0,
      "logps/rejected": -960.0,
      "loss": 0.2388,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.21875,
      "rewards/margins": 2.609375,
      "rewards/rejected": -6.84375,
      "step": 1620
    },
    {
      "epoch": 1.705913134484563,
      "grad_norm": 3.5283350333951633,
      "learning_rate": 4.164478775004775e-05,
      "logits/chosen": -5.65625,
      "logits/rejected": -3.703125,
      "logps/chosen": -764.0,
      "logps/rejected": -1012.0,
      "loss": 0.2165,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.6875,
      "rewards/margins": 2.6875,
      "rewards/rejected": -7.34375,
      "step": 1630
    },
    {
      "epoch": 1.716378859236002,
      "grad_norm": 2.8779850397285642,
      "learning_rate": 4.150796634186784e-05,
      "logits/chosen": -6.5,
      "logits/rejected": -4.59375,
      "logps/chosen": -828.0,
      "logps/rejected": -1152.0,
      "loss": 0.218,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.71875,
      "rewards/margins": 3.390625,
      "rewards/rejected": -8.125,
      "step": 1640
    },
    {
      "epoch": 1.7268445839874411,
      "grad_norm": 2.481408886713114,
      "learning_rate": 4.1370262543483595e-05,
      "logits/chosen": -6.28125,
      "logits/rejected": -4.25,
      "logps/chosen": -808.0,
      "logps/rejected": -1064.0,
      "loss": 0.2176,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.75,
      "rewards/margins": 2.9375,
      "rewards/rejected": -7.6875,
      "step": 1650
    },
    {
      "epoch": 1.7373103087388801,
      "grad_norm": 2.1094531143461777,
      "learning_rate": 4.123168371549175e-05,
      "logits/chosen": -4.3125,
      "logits/rejected": -3.015625,
      "logps/chosen": -864.0,
      "logps/rejected": -1120.0,
      "loss": 0.2373,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -5.5,
      "rewards/margins": 2.890625,
      "rewards/rejected": -8.375,
      "step": 1660
    },
    {
      "epoch": 1.7477760334903192,
      "grad_norm": 2.081186434782386,
      "learning_rate": 4.1092237265261426e-05,
      "logits/chosen": -5.0625,
      "logits/rejected": -2.703125,
      "logps/chosen": -888.0,
      "logps/rejected": -1128.0,
      "loss": 0.2195,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.59375,
      "rewards/margins": 2.828125,
      "rewards/rejected": -8.4375,
      "step": 1670
    },
    {
      "epoch": 1.7582417582417582,
      "grad_norm": 3.7374931598597065,
      "learning_rate": 4.095193064653823e-05,
      "logits/chosen": -4.46875,
      "logits/rejected": -2.578125,
      "logps/chosen": -828.0,
      "logps/rejected": -1080.0,
      "loss": 0.2421,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.4375,
      "rewards/margins": 2.8125,
      "rewards/rejected": -8.25,
      "step": 1680
    },
    {
      "epoch": 1.7687074829931972,
      "grad_norm": 2.5735549072900583,
      "learning_rate": 4.081077135904584e-05,
      "logits/chosen": -6.34375,
      "logits/rejected": -3.703125,
      "logps/chosen": -920.0,
      "logps/rejected": -1104.0,
      "loss": 0.217,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.71875,
      "rewards/margins": 2.625,
      "rewards/rejected": -8.375,
      "step": 1690
    },
    {
      "epoch": 1.7791732077446363,
      "grad_norm": 2.9795017126662375,
      "learning_rate": 4.066876694808512e-05,
      "logits/chosen": -6.3125,
      "logits/rejected": -4.46875,
      "logps/chosen": -836.0,
      "logps/rejected": -1072.0,
      "loss": 0.2327,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -5.25,
      "rewards/margins": 2.640625,
      "rewards/rejected": -7.875,
      "step": 1700
    },
    {
      "epoch": 1.7791732077446363,
      "eval_logits/chosen": -6.0625,
      "eval_logits/rejected": -4.5,
      "eval_logps/chosen": -908.0,
      "eval_logps/rejected": -1000.0,
      "eval_loss": 0.6141093969345093,
      "eval_rewards/accuracies": 0.716796875,
      "eval_rewards/chosen": -5.9375,
      "eval_rewards/margins": 1.21875,
      "eval_rewards/rejected": -7.15625,
      "eval_runtime": 52.8573,
      "eval_samples_per_second": 37.838,
      "eval_steps_per_second": 0.605,
      "step": 1700
    },
    {
      "epoch": 1.7896389324960753,
      "grad_norm": 2.246296295697,
      "learning_rate": 4.052592500413079e-05,
      "logits/chosen": -6.53125,
      "logits/rejected": -4.5625,
      "logps/chosen": -892.0,
      "logps/rejected": -1064.0,
      "loss": 0.2642,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.28125,
      "rewards/margins": 2.578125,
      "rewards/rejected": -7.875,
      "step": 1710
    },
    {
      "epoch": 1.8001046572475143,
      "grad_norm": 1.5806277059274703,
      "learning_rate": 4.0382253162425696e-05,
      "logits/chosen": -6.96875,
      "logits/rejected": -5.46875,
      "logps/chosen": -812.0,
      "logps/rejected": -1008.0,
      "loss": 0.2702,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.78125,
      "rewards/margins": 2.421875,
      "rewards/rejected": -7.1875,
      "step": 1720
    },
    {
      "epoch": 1.8105703819989536,
      "grad_norm": 2.273534363986886,
      "learning_rate": 4.023775910257271e-05,
      "logits/chosen": -6.96875,
      "logits/rejected": -4.46875,
      "logps/chosen": -844.0,
      "logps/rejected": -1080.0,
      "loss": 0.2014,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.09375,
      "rewards/margins": 2.890625,
      "rewards/rejected": -7.96875,
      "step": 1730
    },
    {
      "epoch": 1.8210361067503924,
      "grad_norm": 3.5238832041647283,
      "learning_rate": 4.0092450548124235e-05,
      "logits/chosen": -5.09375,
      "logits/rejected": -2.28125,
      "logps/chosen": -1032.0,
      "logps/rejected": -1288.0,
      "loss": 0.2676,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.84375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -10.0,
      "step": 1740
    },
    {
      "epoch": 1.8315018315018317,
      "grad_norm": 3.6999864745213786,
      "learning_rate": 3.994633526616936e-05,
      "logits/chosen": -6.53125,
      "logits/rejected": -3.46875,
      "logps/chosen": -944.0,
      "logps/rejected": -1144.0,
      "loss": 0.2675,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.03125,
      "rewards/margins": 2.671875,
      "rewards/rejected": -8.6875,
      "step": 1750
    },
    {
      "epoch": 1.8419675562532705,
      "grad_norm": 2.2019977227457685,
      "learning_rate": 3.9799421066918653e-05,
      "logits/chosen": -7.71875,
      "logits/rejected": -6.1875,
      "logps/chosen": -840.0,
      "logps/rejected": -1056.0,
      "loss": 0.2259,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.28125,
      "rewards/margins": 2.4375,
      "rewards/rejected": -7.71875,
      "step": 1760
    },
    {
      "epoch": 1.8524332810047097,
      "grad_norm": 1.7022096305520797,
      "learning_rate": 3.965171580328675e-05,
      "logits/chosen": -6.8125,
      "logits/rejected": -5.1875,
      "logps/chosen": -888.0,
      "logps/rejected": -1104.0,
      "loss": 0.2178,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.34375,
      "rewards/margins": 2.734375,
      "rewards/rejected": -8.0625,
      "step": 1770
    },
    {
      "epoch": 1.8628990057561485,
      "grad_norm": 2.5728669879495243,
      "learning_rate": 3.950322737047254e-05,
      "logits/chosen": -6.5,
      "logits/rejected": -4.4375,
      "logps/chosen": -940.0,
      "logps/rejected": -1208.0,
      "loss": 0.2442,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.71875,
      "rewards/margins": 3.046875,
      "rewards/rejected": -8.8125,
      "step": 1780
    },
    {
      "epoch": 1.8733647305075878,
      "grad_norm": 3.3021132407979272,
      "learning_rate": 3.93539637055372e-05,
      "logits/chosen": -6.0,
      "logits/rejected": -3.84375,
      "logps/chosen": -896.0,
      "logps/rejected": -1120.0,
      "loss": 0.2241,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.71875,
      "rewards/margins": 2.703125,
      "rewards/rejected": -8.4375,
      "step": 1790
    },
    {
      "epoch": 1.8838304552590266,
      "grad_norm": 3.5449002725785066,
      "learning_rate": 3.9203932786979904e-05,
      "logits/chosen": -4.875,
      "logits/rejected": -2.703125,
      "logps/chosen": -848.0,
      "logps/rejected": -1104.0,
      "loss": 0.2221,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.4375,
      "rewards/margins": 2.9375,
      "rewards/rejected": -8.375,
      "step": 1800
    },
    {
      "epoch": 1.8838304552590266,
      "eval_logits/chosen": -3.625,
      "eval_logits/rejected": -1.984375,
      "eval_logps/chosen": -960.0,
      "eval_logps/rejected": -1048.0,
      "eval_loss": 0.607171893119812,
      "eval_rewards/accuracies": 0.7265625,
      "eval_rewards/chosen": -6.46875,
      "eval_rewards/margins": 1.1875,
      "eval_rewards/rejected": -7.65625,
      "eval_runtime": 52.8636,
      "eval_samples_per_second": 37.833,
      "eval_steps_per_second": 0.605,
      "step": 1800
    },
    {
      "epoch": 1.8942961800104658,
      "grad_norm": 2.2325981090247087,
      "learning_rate": 3.905314263431136e-05,
      "logits/chosen": -2.609375,
      "logits/rejected": -0.62890625,
      "logps/chosen": -912.0,
      "logps/rejected": -1192.0,
      "loss": 0.2316,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.15625,
      "rewards/margins": 3.03125,
      "rewards/rejected": -9.1875,
      "step": 1810
    },
    {
      "epoch": 1.9047619047619047,
      "grad_norm": 2.755081964679097,
      "learning_rate": 3.8901601307625144e-05,
      "logits/chosen": -2.609375,
      "logits/rejected": -0.63671875,
      "logps/chosen": -948.0,
      "logps/rejected": -1176.0,
      "loss": 0.2322,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -6.4375,
      "rewards/margins": 2.640625,
      "rewards/rejected": -9.0625,
      "step": 1820
    },
    {
      "epoch": 1.915227629513344,
      "grad_norm": 2.81171176700697,
      "learning_rate": 3.87493169071669e-05,
      "logits/chosen": -3.78125,
      "logits/rejected": -2.3125,
      "logps/chosen": -796.0,
      "logps/rejected": -1072.0,
      "loss": 0.2409,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.125,
      "rewards/margins": 2.640625,
      "rewards/rejected": -7.78125,
      "step": 1830
    },
    {
      "epoch": 1.9256933542647827,
      "grad_norm": 2.959505766153652,
      "learning_rate": 3.859629757290132e-05,
      "logits/chosen": -5.625,
      "logits/rejected": -3.234375,
      "logps/chosen": -848.0,
      "logps/rejected": -1056.0,
      "loss": 0.2436,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.90625,
      "rewards/margins": 2.75,
      "rewards/rejected": -7.65625,
      "step": 1840
    },
    {
      "epoch": 1.936159079016222,
      "grad_norm": 2.224627550716824,
      "learning_rate": 3.8442551484077055e-05,
      "logits/chosen": -5.375,
      "logits/rejected": -3.21875,
      "logps/chosen": -836.0,
      "logps/rejected": -1080.0,
      "loss": 0.2241,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.1875,
      "rewards/margins": 2.75,
      "rewards/rejected": -7.9375,
      "step": 1850
    },
    {
      "epoch": 1.9466248037676608,
      "grad_norm": 2.450233635234575,
      "learning_rate": 3.828808685878955e-05,
      "logits/chosen": -4.59375,
      "logits/rejected": -2.328125,
      "logps/chosen": -872.0,
      "logps/rejected": -1088.0,
      "loss": 0.2398,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.625,
      "rewards/margins": 2.71875,
      "rewards/rejected": -8.375,
      "step": 1860
    },
    {
      "epoch": 1.9570905285191,
      "grad_norm": 2.25540433870103,
      "learning_rate": 3.8132911953541715e-05,
      "logits/chosen": -4.75,
      "logits/rejected": -1.8828125,
      "logps/chosen": -908.0,
      "logps/rejected": -1144.0,
      "loss": 0.202,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.34375,
      "rewards/margins": 3.109375,
      "rewards/rejected": -8.5,
      "step": 1870
    },
    {
      "epoch": 1.9675562532705388,
      "grad_norm": 2.52329283794622,
      "learning_rate": 3.7977035062802636e-05,
      "logits/chosen": -5.21875,
      "logits/rejected": -3.125,
      "logps/chosen": -844.0,
      "logps/rejected": -1080.0,
      "loss": 0.2225,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.1875,
      "rewards/margins": 2.9375,
      "rewards/rejected": -8.125,
      "step": 1880
    },
    {
      "epoch": 1.978021978021978,
      "grad_norm": 2.820473355276973,
      "learning_rate": 3.782046451856417e-05,
      "logits/chosen": -6.75,
      "logits/rejected": -4.59375,
      "logps/chosen": -816.0,
      "logps/rejected": -1056.0,
      "loss": 0.2529,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -4.9375,
      "rewards/margins": 2.8125,
      "rewards/rejected": -7.71875,
      "step": 1890
    },
    {
      "epoch": 1.988487702773417,
      "grad_norm": 2.7800488605542393,
      "learning_rate": 3.7663208689895654e-05,
      "logits/chosen": -5.96875,
      "logits/rejected": -4.25,
      "logps/chosen": -904.0,
      "logps/rejected": -1144.0,
      "loss": 0.2153,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.59375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -8.375,
      "step": 1900
    },
    {
      "epoch": 1.988487702773417,
      "eval_logits/chosen": -4.875,
      "eval_logits/rejected": -3.3125,
      "eval_logps/chosen": -964.0,
      "eval_logps/rejected": -1056.0,
      "eval_loss": 0.594921886920929,
      "eval_rewards/accuracies": 0.7265625,
      "eval_rewards/chosen": -6.5,
      "eval_rewards/margins": 1.1953125,
      "eval_rewards/rejected": -7.6875,
      "eval_runtime": 52.89,
      "eval_samples_per_second": 37.814,
      "eval_steps_per_second": 0.605,
      "step": 1900
    },
    {
      "epoch": 1.9989534275248562,
      "grad_norm": 2.7990258770794254,
      "learning_rate": 3.750527598249647e-05,
      "logits/chosen": -5.1875,
      "logits/rejected": -3.125,
      "logps/chosen": -928.0,
      "logps/rejected": -1152.0,
      "loss": 0.2225,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.875,
      "rewards/margins": 2.828125,
      "rewards/rejected": -8.6875,
      "step": 1910
    },
    {
      "epoch": 2.009419152276295,
      "grad_norm": 0.33032701679760734,
      "learning_rate": 3.734667483824681e-05,
      "logits/chosen": -6.125,
      "logits/rejected": -2.78125,
      "logps/chosen": -892.0,
      "logps/rejected": -1312.0,
      "loss": 0.0487,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.28125,
      "rewards/margins": 4.875,
      "rewards/rejected": -10.1875,
      "step": 1920
    },
    {
      "epoch": 2.0198848770277342,
      "grad_norm": 0.7845585783913027,
      "learning_rate": 3.718741373475641e-05,
      "logits/chosen": -4.21875,
      "logits/rejected": -0.78515625,
      "logps/chosen": -900.0,
      "logps/rejected": -1424.0,
      "loss": 0.0395,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.0625,
      "rewards/margins": 5.4375,
      "rewards/rejected": -11.5,
      "step": 1930
    },
    {
      "epoch": 2.030350601779173,
      "grad_norm": 0.5975494289154536,
      "learning_rate": 3.7027501184911385e-05,
      "logits/chosen": -4.03125,
      "logits/rejected": 0.1279296875,
      "logps/chosen": -984.0,
      "logps/rejected": -1552.0,
      "loss": 0.0278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.75,
      "rewards/margins": 6.09375,
      "rewards/rejected": -12.875,
      "step": 1940
    },
    {
      "epoch": 2.0408163265306123,
      "grad_norm": 0.6075478455420598,
      "learning_rate": 3.6866945736419234e-05,
      "logits/chosen": -3.75,
      "logits/rejected": 0.6953125,
      "logps/chosen": -1004.0,
      "logps/rejected": -1592.0,
      "loss": 0.0341,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.96875,
      "rewards/margins": 6.34375,
      "rewards/rejected": -13.3125,
      "step": 1950
    },
    {
      "epoch": 2.051282051282051,
      "grad_norm": 0.5887917714033745,
      "learning_rate": 3.67057559713519e-05,
      "logits/chosen": -3.1875,
      "logits/rejected": -0.80859375,
      "logps/chosen": -924.0,
      "logps/rejected": -1560.0,
      "loss": 0.0278,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.40625,
      "rewards/margins": 6.28125,
      "rewards/rejected": -12.6875,
      "step": 1960
    },
    {
      "epoch": 2.0617477760334904,
      "grad_norm": 1.663357092082489,
      "learning_rate": 3.6543940505687096e-05,
      "logits/chosen": -3.75,
      "logits/rejected": 0.06201171875,
      "logps/chosen": -1008.0,
      "logps/rejected": -1600.0,
      "loss": 0.0317,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.8125,
      "rewards/margins": 6.34375,
      "rewards/rejected": -13.125,
      "step": 1970
    },
    {
      "epoch": 2.072213500784929,
      "grad_norm": 3.371311980994405,
      "learning_rate": 3.638150798884768e-05,
      "logits/chosen": -3.515625,
      "logits/rejected": -0.2392578125,
      "logps/chosen": -968.0,
      "logps/rejected": -1568.0,
      "loss": 0.0309,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.4375,
      "rewards/margins": 6.375,
      "rewards/rejected": -12.8125,
      "step": 1980
    },
    {
      "epoch": 2.0826792255363684,
      "grad_norm": 0.9209075646693089,
      "learning_rate": 3.621846710323941e-05,
      "logits/chosen": -3.25,
      "logits/rejected": 0.0576171875,
      "logps/chosen": -956.0,
      "logps/rejected": -1528.0,
      "loss": 0.0295,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.25,
      "rewards/margins": 6.21875,
      "rewards/rejected": -12.4375,
      "step": 1990
    },
    {
      "epoch": 2.0931449502878072,
      "grad_norm": 0.3031417723240036,
      "learning_rate": 3.605482656378677e-05,
      "logits/chosen": -3.65625,
      "logits/rejected": 0.36328125,
      "logps/chosen": -912.0,
      "logps/rejected": -1528.0,
      "loss": 0.0215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.125,
      "rewards/margins": 6.5625,
      "rewards/rejected": -12.6875,
      "step": 2000
    },
    {
      "epoch": 2.0931449502878072,
      "eval_logits/chosen": -1.9609375,
      "eval_logits/rejected": -0.107421875,
      "eval_logps/chosen": -1184.0,
      "eval_logps/rejected": -1336.0,
      "eval_loss": 0.746999979019165,
      "eval_rewards/accuracies": 0.724609375,
      "eval_rewards/chosen": -8.6875,
      "eval_rewards/margins": 1.8125,
      "eval_rewards/rejected": -10.5,
      "eval_runtime": 52.8881,
      "eval_samples_per_second": 37.816,
      "eval_steps_per_second": 0.605,
      "step": 2000
    },
    {
      "epoch": 2.1036106750392465,
      "grad_norm": 1.9374058291925411,
      "learning_rate": 3.589059511746722e-05,
      "logits/chosen": -2.953125,
      "logits/rejected": 0.369140625,
      "logps/chosen": -1096.0,
      "logps/rejected": -1768.0,
      "loss": 0.0289,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.5625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -14.625,
      "step": 2010
    },
    {
      "epoch": 2.1140763997906853,
      "grad_norm": 0.4050763470400383,
      "learning_rate": 3.572578154284356e-05,
      "logits/chosen": -3.5,
      "logits/rejected": 0.047119140625,
      "logps/chosen": -1096.0,
      "logps/rejected": -1784.0,
      "loss": 0.0228,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.75,
      "rewards/margins": 7.21875,
      "rewards/rejected": -15.0,
      "step": 2020
    },
    {
      "epoch": 2.1245421245421245,
      "grad_norm": 0.4988691556623063,
      "learning_rate": 3.556039464959474e-05,
      "logits/chosen": -4.375,
      "logits/rejected": -0.333984375,
      "logps/chosen": -1040.0,
      "logps/rejected": -1680.0,
      "loss": 0.0285,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.125,
      "rewards/margins": 6.90625,
      "rewards/rejected": -14.0,
      "step": 2030
    },
    {
      "epoch": 2.1350078492935634,
      "grad_norm": 0.3314005859234191,
      "learning_rate": 3.539444327804498e-05,
      "logits/chosen": -3.390625,
      "logits/rejected": -0.671875,
      "logps/chosen": -1032.0,
      "logps/rejected": -1696.0,
      "loss": 0.0305,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.3125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -14.0625,
      "step": 2040
    },
    {
      "epoch": 2.1454735740450026,
      "grad_norm": 0.5736427946721916,
      "learning_rate": 3.5227936298691214e-05,
      "logits/chosen": -2.546875,
      "logits/rejected": 0.8984375,
      "logps/chosen": -1088.0,
      "logps/rejected": -1704.0,
      "loss": 0.037,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -7.71875,
      "rewards/margins": 6.5625,
      "rewards/rejected": -14.25,
      "step": 2050
    },
    {
      "epoch": 2.155939298796442,
      "grad_norm": 1.3303976437021603,
      "learning_rate": 3.5060882611728915e-05,
      "logits/chosen": -3.5625,
      "logits/rejected": -0.2734375,
      "logps/chosen": -976.0,
      "logps/rejected": -1544.0,
      "loss": 0.0293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.3125,
      "rewards/margins": 6.1875,
      "rewards/rejected": -12.5,
      "step": 2060
    },
    {
      "epoch": 2.1664050235478807,
      "grad_norm": 0.3792335895581482,
      "learning_rate": 3.4893291146576404e-05,
      "logits/chosen": -2.8125,
      "logits/rejected": 0.12890625,
      "logps/chosen": -940.0,
      "logps/rejected": -1576.0,
      "loss": 0.0204,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.375,
      "rewards/margins": 6.5,
      "rewards/rejected": -12.875,
      "step": 2070
    },
    {
      "epoch": 2.17687074829932,
      "grad_norm": 1.0854147028131242,
      "learning_rate": 3.4725170861397524e-05,
      "logits/chosen": -4.59375,
      "logits/rejected": -0.474609375,
      "logps/chosen": -948.0,
      "logps/rejected": -1584.0,
      "loss": 0.0327,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.09375,
      "rewards/margins": 6.71875,
      "rewards/rejected": -12.8125,
      "step": 2080
    },
    {
      "epoch": 2.1873364730507587,
      "grad_norm": 1.263333596618435,
      "learning_rate": 3.45565307426228e-05,
      "logits/chosen": -2.125,
      "logits/rejected": 1.1484375,
      "logps/chosen": -1136.0,
      "logps/rejected": -1840.0,
      "loss": 0.0321,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.90625,
      "rewards/margins": 7.5,
      "rewards/rejected": -15.375,
      "step": 2090
    },
    {
      "epoch": 2.197802197802198,
      "grad_norm": 1.206205770100736,
      "learning_rate": 3.438737980446912e-05,
      "logits/chosen": -2.296875,
      "logits/rejected": 1.296875,
      "logps/chosen": -1104.0,
      "logps/rejected": -1768.0,
      "loss": 0.0303,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.6875,
      "rewards/margins": 7.125,
      "rewards/rejected": -14.8125,
      "step": 2100
    },
    {
      "epoch": 2.197802197802198,
      "eval_logits/chosen": -2.078125,
      "eval_logits/rejected": -0.1298828125,
      "eval_logps/chosen": -1144.0,
      "eval_logps/rejected": -1312.0,
      "eval_loss": 0.7468593716621399,
      "eval_rewards/accuracies": 0.703125,
      "eval_rewards/chosen": -8.3125,
      "eval_rewards/margins": 1.9453125,
      "eval_rewards/rejected": -10.25,
      "eval_runtime": 52.9072,
      "eval_samples_per_second": 37.802,
      "eval_steps_per_second": 0.605,
      "step": 2100
    },
    {
      "epoch": 2.208267922553637,
      "grad_norm": 0.4320527786696419,
      "learning_rate": 3.4217727088457877e-05,
      "logits/chosen": -4.0625,
      "logits/rejected": -0.01708984375,
      "logps/chosen": -936.0,
      "logps/rejected": -1536.0,
      "loss": 0.0349,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.0,
      "rewards/margins": 6.53125,
      "rewards/rejected": -12.5,
      "step": 2110
    },
    {
      "epoch": 2.218733647305076,
      "grad_norm": 2.2654140252539956,
      "learning_rate": 3.40475816629317e-05,
      "logits/chosen": -3.859375,
      "logits/rejected": -0.404296875,
      "logps/chosen": -952.0,
      "logps/rejected": -1680.0,
      "loss": 0.0392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.3125,
      "rewards/margins": 7.59375,
      "rewards/rejected": -13.875,
      "step": 2120
    },
    {
      "epoch": 2.229199372056515,
      "grad_norm": 0.48033075629807964,
      "learning_rate": 3.387695262256969e-05,
      "logits/chosen": -4.8125,
      "logits/rejected": -0.97265625,
      "logps/chosen": -920.0,
      "logps/rejected": -1568.0,
      "loss": 0.0295,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.875,
      "rewards/margins": 6.8125,
      "rewards/rejected": -12.6875,
      "step": 2130
    },
    {
      "epoch": 2.239665096807954,
      "grad_norm": 1.3227057399070403,
      "learning_rate": 3.3705849087901345e-05,
      "logits/chosen": -5.625,
      "logits/rejected": -2.234375,
      "logps/chosen": -912.0,
      "logps/rejected": -1576.0,
      "loss": 0.0248,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.03125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -12.8125,
      "step": 2140
    },
    {
      "epoch": 2.250130821559393,
      "grad_norm": 2.6922979427204625,
      "learning_rate": 3.3534280204819004e-05,
      "logits/chosen": -5.3125,
      "logits/rejected": -1.046875,
      "logps/chosen": -1168.0,
      "logps/rejected": -1840.0,
      "loss": 0.0333,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.84375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -15.25,
      "step": 2150
    },
    {
      "epoch": 2.260596546310832,
      "grad_norm": 0.5164523934974737,
      "learning_rate": 3.3362255144088986e-05,
      "logits/chosen": -4.21875,
      "logits/rejected": -0.89453125,
      "logps/chosen": -1048.0,
      "logps/rejected": -1736.0,
      "loss": 0.0325,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.34375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -14.3125,
      "step": 2160
    },
    {
      "epoch": 2.271062271062271,
      "grad_norm": 2.6614586233503834,
      "learning_rate": 3.3189783100861416e-05,
      "logits/chosen": -4.375,
      "logits/rejected": -0.177734375,
      "logps/chosen": -1064.0,
      "logps/rejected": -1672.0,
      "loss": 0.0356,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.15625,
      "rewards/margins": 6.84375,
      "rewards/rejected": -14.0,
      "step": 2170
    },
    {
      "epoch": 2.2815279958137102,
      "grad_norm": 0.4538506307609594,
      "learning_rate": 3.301687329417866e-05,
      "logits/chosen": -4.53125,
      "logits/rejected": -0.486328125,
      "logps/chosen": -1032.0,
      "logps/rejected": -1624.0,
      "loss": 0.0299,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.625,
      "rewards/margins": 6.5625,
      "rewards/rejected": -13.1875,
      "step": 2180
    },
    {
      "epoch": 2.291993720565149,
      "grad_norm": 1.358570693850712,
      "learning_rate": 3.284353496648262e-05,
      "logits/chosen": -3.96875,
      "logits/rejected": -0.007110595703125,
      "logps/chosen": -992.0,
      "logps/rejected": -1736.0,
      "loss": 0.0337,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.78125,
      "rewards/margins": 7.34375,
      "rewards/rejected": -14.125,
      "step": 2190
    },
    {
      "epoch": 2.3024594453165883,
      "grad_norm": 2.0759035062148037,
      "learning_rate": 3.2669777383120667e-05,
      "logits/chosen": -3.53125,
      "logits/rejected": 0.56640625,
      "logps/chosen": -1032.0,
      "logps/rejected": -1640.0,
      "loss": 0.0322,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -7.15625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -13.9375,
      "step": 2200
    },
    {
      "epoch": 2.3024594453165883,
      "eval_logits/chosen": -2.609375,
      "eval_logits/rejected": -0.515625,
      "eval_logps/chosen": -1168.0,
      "eval_logps/rejected": -1328.0,
      "eval_loss": 0.7583593726158142,
      "eval_rewards/accuracies": 0.7109375,
      "eval_rewards/chosen": -8.5625,
      "eval_rewards/margins": 1.8828125,
      "eval_rewards/rejected": -10.4375,
      "eval_runtime": 52.8854,
      "eval_samples_per_second": 37.818,
      "eval_steps_per_second": 0.605,
      "step": 2200
    },
    {
      "epoch": 2.312925170068027,
      "grad_norm": 1.0386695661847614,
      "learning_rate": 3.2495609831850355e-05,
      "logits/chosen": -3.25,
      "logits/rejected": 1.03125,
      "logps/chosen": -996.0,
      "logps/rejected": -1576.0,
      "loss": 0.0291,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.78125,
      "rewards/margins": 6.375,
      "rewards/rejected": -13.125,
      "step": 2210
    },
    {
      "epoch": 2.3233908948194664,
      "grad_norm": 0.9017315734991943,
      "learning_rate": 3.232104162234305e-05,
      "logits/chosen": -4.09375,
      "logits/rejected": -0.050048828125,
      "logps/chosen": -964.0,
      "logps/rejected": -1600.0,
      "loss": 0.0237,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -12.9375,
      "step": 2220
    },
    {
      "epoch": 2.333856619570905,
      "grad_norm": 3.5221813476753634,
      "learning_rate": 3.214608208568621e-05,
      "logits/chosen": -3.484375,
      "logits/rejected": 0.294921875,
      "logps/chosen": -944.0,
      "logps/rejected": -1568.0,
      "loss": 0.0343,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.25,
      "rewards/margins": 6.625,
      "rewards/rejected": -12.875,
      "step": 2230
    },
    {
      "epoch": 2.3443223443223444,
      "grad_norm": 0.8215146677057751,
      "learning_rate": 3.197074057388471e-05,
      "logits/chosen": -5.03125,
      "logits/rejected": -0.55859375,
      "logps/chosen": -816.0,
      "logps/rejected": -1544.0,
      "loss": 0.0267,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 7.375,
      "rewards/rejected": -12.625,
      "step": 2240
    },
    {
      "epoch": 2.3547880690737832,
      "grad_norm": 1.3572854776003833,
      "learning_rate": 3.179502645936085e-05,
      "logits/chosen": -3.25,
      "logits/rejected": 0.8125,
      "logps/chosen": -1056.0,
      "logps/rejected": -1704.0,
      "loss": 0.0257,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.21875,
      "rewards/margins": 7.0,
      "rewards/rejected": -14.25,
      "step": 2250
    },
    {
      "epoch": 2.3652537938252225,
      "grad_norm": 1.5402415847464914,
      "learning_rate": 3.16189491344535e-05,
      "logits/chosen": -3.6875,
      "logits/rejected": -0.07373046875,
      "logps/chosen": -1012.0,
      "logps/rejected": -1664.0,
      "loss": 0.0341,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.4375,
      "rewards/margins": 7.0,
      "rewards/rejected": -13.4375,
      "step": 2260
    },
    {
      "epoch": 2.3757195185766613,
      "grad_norm": 0.7612792883410395,
      "learning_rate": 3.144251801091594e-05,
      "logits/chosen": -3.921875,
      "logits/rejected": 0.173828125,
      "logps/chosen": -916.0,
      "logps/rejected": -1552.0,
      "loss": 0.0293,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.75,
      "rewards/margins": 6.875,
      "rewards/rejected": -12.625,
      "step": 2270
    },
    {
      "epoch": 2.3861852433281006,
      "grad_norm": 0.36793712218726776,
      "learning_rate": 3.126574251941288e-05,
      "logits/chosen": -3.46875,
      "logits/rejected": -0.1416015625,
      "logps/chosen": -1020.0,
      "logps/rejected": -1704.0,
      "loss": 0.0229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.78125,
      "rewards/margins": 7.0625,
      "rewards/rejected": -13.8125,
      "step": 2280
    },
    {
      "epoch": 2.3966509680795394,
      "grad_norm": 0.3368205121436285,
      "learning_rate": 3.108863210901626e-05,
      "logits/chosen": -2.453125,
      "logits/rejected": 0.765625,
      "logps/chosen": -1032.0,
      "logps/rejected": -1720.0,
      "loss": 0.0319,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.96875,
      "rewards/margins": 7.1875,
      "rewards/rejected": -14.1875,
      "step": 2290
    },
    {
      "epoch": 2.4071166928309786,
      "grad_norm": 0.3793151914230102,
      "learning_rate": 3.091119624670027e-05,
      "logits/chosen": -1.71875,
      "logits/rejected": 1.4296875,
      "logps/chosen": -1104.0,
      "logps/rejected": -1800.0,
      "loss": 0.0253,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.8125,
      "rewards/margins": 7.28125,
      "rewards/rejected": -15.0625,
      "step": 2300
    },
    {
      "epoch": 2.4071166928309786,
      "eval_logits/chosen": -0.75390625,
      "eval_logits/rejected": 1.265625,
      "eval_logps/chosen": -1296.0,
      "eval_logps/rejected": -1480.0,
      "eval_loss": 0.808718740940094,
      "eval_rewards/accuracies": 0.712890625,
      "eval_rewards/chosen": -9.8125,
      "eval_rewards/margins": 2.125,
      "eval_rewards/rejected": -11.9375,
      "eval_runtime": 52.9002,
      "eval_samples_per_second": 37.807,
      "eval_steps_per_second": 0.605,
      "step": 2300
    },
    {
      "epoch": 2.4175824175824174,
      "grad_norm": 1.0436093492415373,
      "learning_rate": 3.0733444416835294e-05,
      "logits/chosen": -2.53125,
      "logits/rejected": 0.76171875,
      "logps/chosen": -1032.0,
      "logps/rejected": -1776.0,
      "loss": 0.0259,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.34375,
      "rewards/margins": 7.4375,
      "rewards/rejected": -14.75,
      "step": 2310
    },
    {
      "epoch": 2.4280481423338567,
      "grad_norm": 0.6929838487811482,
      "learning_rate": 3.0555386120680915e-05,
      "logits/chosen": -2.96875,
      "logits/rejected": 0.921875,
      "logps/chosen": -1080.0,
      "logps/rejected": -1752.0,
      "loss": 0.0273,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.125,
      "rewards/margins": 7.34375,
      "rewards/rejected": -14.4375,
      "step": 2320
    },
    {
      "epoch": 2.4385138670852955,
      "grad_norm": 0.8425811766791768,
      "learning_rate": 3.0377030875878064e-05,
      "logits/chosen": -2.09375,
      "logits/rejected": 1.390625,
      "logps/chosen": -1080.0,
      "logps/rejected": -1768.0,
      "loss": 0.0352,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.3125,
      "rewards/margins": 7.5,
      "rewards/rejected": -14.8125,
      "step": 2330
    },
    {
      "epoch": 2.4489795918367347,
      "grad_norm": 0.23250126941693725,
      "learning_rate": 3.0198388215940305e-05,
      "logits/chosen": -3.59375,
      "logits/rejected": 0.267578125,
      "logps/chosen": -1104.0,
      "logps/rejected": -1704.0,
      "loss": 0.0176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.03125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -13.8125,
      "step": 2340
    },
    {
      "epoch": 2.4594453165881736,
      "grad_norm": 0.34696574142214337,
      "learning_rate": 3.0019467689744217e-05,
      "logits/chosen": -2.203125,
      "logits/rejected": 1.7421875,
      "logps/chosen": -1004.0,
      "logps/rejected": -1720.0,
      "loss": 0.0219,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.90625,
      "rewards/margins": 7.5,
      "rewards/rejected": -14.375,
      "step": 2350
    },
    {
      "epoch": 2.469911041339613,
      "grad_norm": 0.7548322157415859,
      "learning_rate": 2.9840278861019e-05,
      "logits/chosen": -1.5859375,
      "logits/rejected": 2.0625,
      "logps/chosen": -1056.0,
      "logps/rejected": -1736.0,
      "loss": 0.0308,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.625,
      "rewards/margins": 7.1875,
      "rewards/rejected": -14.8125,
      "step": 2360
    },
    {
      "epoch": 2.4803767660910516,
      "grad_norm": 2.0665349230040193,
      "learning_rate": 2.9660831307835257e-05,
      "logits/chosen": -0.94140625,
      "logits/rejected": 2.21875,
      "logps/chosen": -1096.0,
      "logps/rejected": -1784.0,
      "loss": 0.0278,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.78125,
      "rewards/margins": 7.28125,
      "rewards/rejected": -15.0625,
      "step": 2370
    },
    {
      "epoch": 2.490842490842491,
      "grad_norm": 3.1198291620615493,
      "learning_rate": 2.9481134622093032e-05,
      "logits/chosen": -0.263671875,
      "logits/rejected": 2.265625,
      "logps/chosen": -1136.0,
      "logps/rejected": -1880.0,
      "loss": 0.0201,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.375,
      "rewards/margins": 7.5625,
      "rewards/rejected": -15.9375,
      "step": 2380
    },
    {
      "epoch": 2.50130821559393,
      "grad_norm": 0.5509370310846502,
      "learning_rate": 2.9301198409009108e-05,
      "logits/chosen": 0.134765625,
      "logits/rejected": 2.953125,
      "logps/chosen": -1112.0,
      "logps/rejected": -1816.0,
      "loss": 0.0286,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -8.0625,
      "rewards/margins": 7.625,
      "rewards/rejected": -15.6875,
      "step": 2390
    },
    {
      "epoch": 2.511773940345369,
      "grad_norm": 3.1272795221082443,
      "learning_rate": 2.9121032286603554e-05,
      "logits/chosen": -1.1953125,
      "logits/rejected": 2.5,
      "logps/chosen": -1112.0,
      "logps/rejected": -1776.0,
      "loss": 0.0302,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.6875,
      "rewards/margins": 7.09375,
      "rewards/rejected": -14.8125,
      "step": 2400
    },
    {
      "epoch": 2.511773940345369,
      "eval_logits/chosen": 0.439453125,
      "eval_logits/rejected": 2.28125,
      "eval_logps/chosen": -1216.0,
      "eval_logps/rejected": -1392.0,
      "eval_loss": 0.8032734394073486,
      "eval_rewards/accuracies": 0.724609375,
      "eval_rewards/chosen": -9.0,
      "eval_rewards/margins": 2.03125,
      "eval_rewards/rejected": -11.0625,
      "eval_runtime": 52.9062,
      "eval_samples_per_second": 37.803,
      "eval_steps_per_second": 0.605,
      "step": 2400
    },
    {
      "epoch": 2.5222396650968077,
      "grad_norm": 0.3983961715251342,
      "learning_rate": 2.894064588518568e-05,
      "logits/chosen": -1.1640625,
      "logits/rejected": 1.6640625,
      "logps/chosen": -1056.0,
      "logps/rejected": -1712.0,
      "loss": 0.0251,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.0625,
      "rewards/margins": 7.125,
      "rewards/rejected": -14.1875,
      "step": 2410
    },
    {
      "epoch": 2.532705389848247,
      "grad_norm": 1.8965194428136705,
      "learning_rate": 2.8760048846839194e-05,
      "logits/chosen": -1.7265625,
      "logits/rejected": 1.3984375,
      "logps/chosen": -968.0,
      "logps/rejected": -1632.0,
      "loss": 0.0283,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.4375,
      "rewards/margins": 7.0,
      "rewards/rejected": -13.4375,
      "step": 2420
    },
    {
      "epoch": 2.5431711145996863,
      "grad_norm": 0.7430147021819409,
      "learning_rate": 2.8579250824906877e-05,
      "logits/chosen": -1.3984375,
      "logits/rejected": 1.765625,
      "logps/chosen": -1088.0,
      "logps/rejected": -1792.0,
      "loss": 0.0232,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.5,
      "rewards/margins": 7.5625,
      "rewards/rejected": -15.0,
      "step": 2430
    },
    {
      "epoch": 2.553636839351125,
      "grad_norm": 2.56696162913765,
      "learning_rate": 2.8398261483474547e-05,
      "logits/chosen": -0.58984375,
      "logits/rejected": 2.96875,
      "logps/chosen": -1088.0,
      "logps/rejected": -1848.0,
      "loss": 0.0344,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.71875,
      "rewards/margins": 7.875,
      "rewards/rejected": -15.5625,
      "step": 2440
    },
    {
      "epoch": 2.564102564102564,
      "grad_norm": 2.730368166501539,
      "learning_rate": 2.8217090496854515e-05,
      "logits/chosen": -1.515625,
      "logits/rejected": 1.2109375,
      "logps/chosen": -1064.0,
      "logps/rejected": -1736.0,
      "loss": 0.0313,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.3125,
      "rewards/margins": 7.09375,
      "rewards/rejected": -14.4375,
      "step": 2450
    },
    {
      "epoch": 2.574568288854003,
      "grad_norm": 1.1722855893135804,
      "learning_rate": 2.8035747549068452e-05,
      "logits/chosen": -1.625,
      "logits/rejected": 1.5546875,
      "logps/chosen": -1088.0,
      "logps/rejected": -1752.0,
      "loss": 0.026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.25,
      "rewards/margins": 7.1875,
      "rewards/rejected": -14.4375,
      "step": 2460
    },
    {
      "epoch": 2.5850340136054424,
      "grad_norm": 0.6287233493076456,
      "learning_rate": 2.7854242333329782e-05,
      "logits/chosen": -1.859375,
      "logits/rejected": 1.2109375,
      "logps/chosen": -1048.0,
      "logps/rejected": -1824.0,
      "loss": 0.0283,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.71875,
      "rewards/margins": 7.75,
      "rewards/rejected": -15.5,
      "step": 2470
    },
    {
      "epoch": 2.595499738356881,
      "grad_norm": 0.8747862515761952,
      "learning_rate": 2.7672584551525527e-05,
      "logits/chosen": -0.8359375,
      "logits/rejected": 1.5078125,
      "logps/chosen": -1120.0,
      "logps/rejected": -1896.0,
      "loss": 0.0321,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -8.0,
      "rewards/margins": 7.625,
      "rewards/rejected": -15.6875,
      "step": 2480
    },
    {
      "epoch": 2.60596546310832,
      "grad_norm": 1.4602269013561426,
      "learning_rate": 2.749078391369771e-05,
      "logits/chosen": -2.15625,
      "logits/rejected": 0.734375,
      "logps/chosen": -1056.0,
      "logps/rejected": -1808.0,
      "loss": 0.0302,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.1875,
      "rewards/margins": 7.5625,
      "rewards/rejected": -14.75,
      "step": 2490
    },
    {
      "epoch": 2.6164311878597593,
      "grad_norm": 2.4880564377763768,
      "learning_rate": 2.730885013752439e-05,
      "logits/chosen": -1.359375,
      "logits/rejected": 1.6171875,
      "logps/chosen": -1056.0,
      "logps/rejected": -1808.0,
      "loss": 0.0218,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.4375,
      "rewards/margins": 7.78125,
      "rewards/rejected": -15.25,
      "step": 2500
    },
    {
      "epoch": 2.6164311878597593,
      "eval_logits/chosen": 0.51953125,
      "eval_logits/rejected": 2.296875,
      "eval_logps/chosen": -1408.0,
      "eval_logps/rejected": -1616.0,
      "eval_loss": 0.8602734208106995,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -11.0,
      "eval_rewards/margins": 2.3125,
      "eval_rewards/rejected": -13.3125,
      "eval_runtime": 52.9043,
      "eval_samples_per_second": 37.804,
      "eval_steps_per_second": 0.605,
      "step": 2500
    },
    {
      "epoch": 2.6268969126111985,
      "grad_norm": 0.9370128586176782,
      "learning_rate": 2.712679294780015e-05,
      "logits/chosen": -1.09375,
      "logits/rejected": 2.125,
      "logps/chosen": -1264.0,
      "logps/rejected": -1952.0,
      "loss": 0.0395,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.1875,
      "rewards/margins": 7.625,
      "rewards/rejected": -16.75,
      "step": 2510
    },
    {
      "epoch": 2.6373626373626373,
      "grad_norm": 0.6235520573693828,
      "learning_rate": 2.6944622075916344e-05,
      "logits/chosen": -0.318359375,
      "logits/rejected": 2.59375,
      "logps/chosen": -1136.0,
      "logps/rejected": -1872.0,
      "loss": 0.0275,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.1875,
      "rewards/margins": 7.65625,
      "rewards/rejected": -15.8125,
      "step": 2520
    },
    {
      "epoch": 2.647828362114076,
      "grad_norm": 1.2117156468852468,
      "learning_rate": 2.6762347259340907e-05,
      "logits/chosen": -1.0546875,
      "logits/rejected": 1.296875,
      "logps/chosen": -1020.0,
      "logps/rejected": -1712.0,
      "loss": 0.0281,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.09375,
      "rewards/margins": 7.09375,
      "rewards/rejected": -14.1875,
      "step": 2530
    },
    {
      "epoch": 2.6582940868655154,
      "grad_norm": 0.3622207393417241,
      "learning_rate": 2.6579978241097865e-05,
      "logits/chosen": -1.1328125,
      "logits/rejected": 2.109375,
      "logps/chosen": -1064.0,
      "logps/rejected": -1744.0,
      "loss": 0.0346,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.3125,
      "rewards/margins": 7.3125,
      "rewards/rejected": -14.625,
      "step": 2540
    },
    {
      "epoch": 2.6687598116169546,
      "grad_norm": 0.5754950910620171,
      "learning_rate": 2.639752476924653e-05,
      "logits/chosen": -0.875,
      "logits/rejected": 2.046875,
      "logps/chosen": -1136.0,
      "logps/rejected": -1856.0,
      "loss": 0.0327,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.0625,
      "rewards/margins": 7.5625,
      "rewards/rejected": -15.625,
      "step": 2550
    },
    {
      "epoch": 2.6792255363683934,
      "grad_norm": 1.7122391719550465,
      "learning_rate": 2.621499659636049e-05,
      "logits/chosen": -1.3125,
      "logits/rejected": 1.4921875,
      "logps/chosen": -1064.0,
      "logps/rejected": -1736.0,
      "loss": 0.0258,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.40625,
      "rewards/margins": 6.90625,
      "rewards/rejected": -14.3125,
      "step": 2560
    },
    {
      "epoch": 2.6896912611198327,
      "grad_norm": 0.5733433265622923,
      "learning_rate": 2.6032403479006263e-05,
      "logits/chosen": -1.421875,
      "logits/rejected": 1.640625,
      "logps/chosen": -1056.0,
      "logps/rejected": -1728.0,
      "loss": 0.0202,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.03125,
      "rewards/margins": 7.21875,
      "rewards/rejected": -14.25,
      "step": 2570
    },
    {
      "epoch": 2.7001569858712715,
      "grad_norm": 0.3121083597914369,
      "learning_rate": 2.584975517722181e-05,
      "logits/chosen": -0.9453125,
      "logits/rejected": 2.421875,
      "logps/chosen": -1176.0,
      "logps/rejected": -1864.0,
      "loss": 0.0417,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.625,
      "rewards/margins": 7.4375,
      "rewards/rejected": -16.125,
      "step": 2580
    },
    {
      "epoch": 2.7106227106227108,
      "grad_norm": 3.6347959815546833,
      "learning_rate": 2.5667061453994834e-05,
      "logits/chosen": -0.50390625,
      "logits/rejected": 2.25,
      "logps/chosen": -1120.0,
      "logps/rejected": -1832.0,
      "loss": 0.0385,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -8.1875,
      "rewards/margins": 7.40625,
      "rewards/rejected": -15.625,
      "step": 2590
    },
    {
      "epoch": 2.7210884353741496,
      "grad_norm": 0.5266099919412788,
      "learning_rate": 2.5484332074740913e-05,
      "logits/chosen": -2.40625,
      "logits/rejected": 0.5703125,
      "logps/chosen": -1056.0,
      "logps/rejected": -1728.0,
      "loss": 0.027,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.375,
      "rewards/margins": 7.125,
      "rewards/rejected": -14.5,
      "step": 2600
    },
    {
      "epoch": 2.7210884353741496,
      "eval_logits/chosen": -0.9609375,
      "eval_logits/rejected": 1.0703125,
      "eval_logps/chosen": -1288.0,
      "eval_logps/rejected": -1488.0,
      "eval_loss": 0.8161718845367432,
      "eval_rewards/accuracies": 0.740234375,
      "eval_rewards/chosen": -9.75,
      "eval_rewards/margins": 2.21875,
      "eval_rewards/rejected": -12.0,
      "eval_runtime": 52.8918,
      "eval_samples_per_second": 37.813,
      "eval_steps_per_second": 0.605,
      "step": 2600
    },
    {
      "epoch": 2.731554160125589,
      "grad_norm": 1.5362016184987102,
      "learning_rate": 2.5301576806781518e-05,
      "logits/chosen": -1.84375,
      "logits/rejected": 1.296875,
      "logps/chosen": -1072.0,
      "logps/rejected": -1800.0,
      "loss": 0.0403,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.75,
      "rewards/margins": 7.4375,
      "rewards/rejected": -15.1875,
      "step": 2610
    },
    {
      "epoch": 2.7420198848770276,
      "grad_norm": 1.1533355773019651,
      "learning_rate": 2.5118805418821967e-05,
      "logits/chosen": -1.8515625,
      "logits/rejected": 1.9609375,
      "logps/chosen": -1088.0,
      "logps/rejected": -1848.0,
      "loss": 0.025,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.875,
      "rewards/margins": 7.8125,
      "rewards/rejected": -15.6875,
      "step": 2620
    },
    {
      "epoch": 2.752485609628467,
      "grad_norm": 0.6534763596898375,
      "learning_rate": 2.4936027680429197e-05,
      "logits/chosen": -1.375,
      "logits/rejected": 1.40625,
      "logps/chosen": -1080.0,
      "logps/rejected": -1848.0,
      "loss": 0.0375,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -15.5625,
      "step": 2630
    },
    {
      "epoch": 2.7629513343799057,
      "grad_norm": 1.2269783671113828,
      "learning_rate": 2.475325336150961e-05,
      "logits/chosen": -1.828125,
      "logits/rejected": 1.9921875,
      "logps/chosen": -1128.0,
      "logps/rejected": -1808.0,
      "loss": 0.0474,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.9375,
      "rewards/margins": 7.3125,
      "rewards/rejected": -15.25,
      "step": 2640
    },
    {
      "epoch": 2.773417059131345,
      "grad_norm": 0.42128890294068566,
      "learning_rate": 2.457049223178681e-05,
      "logits/chosen": -1.703125,
      "logits/rejected": 1.9609375,
      "logps/chosen": -1104.0,
      "logps/rejected": -1808.0,
      "loss": 0.0216,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.8125,
      "rewards/margins": 7.4375,
      "rewards/rejected": -15.25,
      "step": 2650
    },
    {
      "epoch": 2.7838827838827838,
      "grad_norm": 1.2008555193471955,
      "learning_rate": 2.438775406027944e-05,
      "logits/chosen": -3.453125,
      "logits/rejected": 0.92578125,
      "logps/chosen": -968.0,
      "logps/rejected": -1608.0,
      "loss": 0.0346,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.6875,
      "rewards/margins": 6.78125,
      "rewards/rejected": -13.4375,
      "step": 2660
    },
    {
      "epoch": 2.794348508634223,
      "grad_norm": 1.6461672196616548,
      "learning_rate": 2.4205048614778926e-05,
      "logits/chosen": -2.15625,
      "logits/rejected": 2.375,
      "logps/chosen": -1012.0,
      "logps/rejected": -1648.0,
      "loss": 0.044,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.03125,
      "rewards/margins": 7.0,
      "rewards/rejected": -14.0625,
      "step": 2670
    },
    {
      "epoch": 2.804814233385662,
      "grad_norm": 1.0824184309184701,
      "learning_rate": 2.402238566132744e-05,
      "logits/chosen": -1.546875,
      "logits/rejected": 1.7734375,
      "logps/chosen": -1112.0,
      "logps/rejected": -1880.0,
      "loss": 0.0184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.59375,
      "rewards/margins": 8.1875,
      "rewards/rejected": -15.75,
      "step": 2680
    },
    {
      "epoch": 2.815279958137101,
      "grad_norm": 1.6308888710356009,
      "learning_rate": 2.383977496369586e-05,
      "logits/chosen": -0.0303955078125,
      "logits/rejected": 2.90625,
      "logps/chosen": -1128.0,
      "logps/rejected": -1912.0,
      "loss": 0.0277,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.25,
      "rewards/margins": 8.1875,
      "rewards/rejected": -16.375,
      "step": 2690
    },
    {
      "epoch": 2.82574568288854,
      "grad_norm": 3.6508001389254043,
      "learning_rate": 2.3657226282861844e-05,
      "logits/chosen": -1.8828125,
      "logits/rejected": 2.171875,
      "logps/chosen": -1128.0,
      "logps/rejected": -1872.0,
      "loss": 0.0274,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.875,
      "rewards/margins": 7.84375,
      "rewards/rejected": -15.75,
      "step": 2700
    },
    {
      "epoch": 2.82574568288854,
      "eval_logits/chosen": -0.91015625,
      "eval_logits/rejected": 1.125,
      "eval_logps/chosen": -1288.0,
      "eval_logps/rejected": -1480.0,
      "eval_loss": 0.8295781016349792,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -9.75,
      "eval_rewards/margins": 2.21875,
      "eval_rewards/rejected": -12.0,
      "eval_runtime": 52.8913,
      "eval_samples_per_second": 37.813,
      "eval_steps_per_second": 0.605,
      "step": 2700
    },
    {
      "epoch": 2.836211407639979,
      "grad_norm": 1.2993477133286373,
      "learning_rate": 2.347474937648811e-05,
      "logits/chosen": -2.671875,
      "logits/rejected": 0.71875,
      "logps/chosen": -1096.0,
      "logps/rejected": -1824.0,
      "loss": 0.0206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 7.875,
      "rewards/rejected": -15.1875,
      "step": 2710
    },
    {
      "epoch": 2.846677132391418,
      "grad_norm": 1.208975326201441,
      "learning_rate": 2.3292353998400875e-05,
      "logits/chosen": -2.453125,
      "logits/rejected": 1.2421875,
      "logps/chosen": -1152.0,
      "logps/rejected": -1928.0,
      "loss": 0.0195,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -8.3125,
      "rewards/margins": 8.25,
      "rewards/rejected": -16.5,
      "step": 2720
    },
    {
      "epoch": 2.857142857142857,
      "grad_norm": 0.2446386189614852,
      "learning_rate": 2.311004989806845e-05,
      "logits/chosen": -2.4375,
      "logits/rejected": 0.9921875,
      "logps/chosen": -1160.0,
      "logps/rejected": -2064.0,
      "loss": 0.0308,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.5625,
      "rewards/margins": 9.0625,
      "rewards/rejected": -17.625,
      "step": 2730
    },
    {
      "epoch": 2.867608581894296,
      "grad_norm": 1.6860858115184227,
      "learning_rate": 2.2927846820080156e-05,
      "logits/chosen": -3.28125,
      "logits/rejected": -0.244140625,
      "logps/chosen": -1072.0,
      "logps/rejected": -1840.0,
      "loss": 0.024,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.5625,
      "rewards/margins": 7.90625,
      "rewards/rejected": -15.4375,
      "step": 2740
    },
    {
      "epoch": 2.8780743066457353,
      "grad_norm": 1.2652848167896822,
      "learning_rate": 2.2745754503625407e-05,
      "logits/chosen": -3.484375,
      "logits/rejected": 0.8203125,
      "logps/chosen": -1016.0,
      "logps/rejected": -1704.0,
      "loss": 0.0242,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.96875,
      "rewards/margins": 7.40625,
      "rewards/rejected": -14.375,
      "step": 2750
    },
    {
      "epoch": 2.8885400313971745,
      "grad_norm": 1.0587944363261317,
      "learning_rate": 2.256378268197317e-05,
      "logits/chosen": -2.875,
      "logits/rejected": 1.1796875,
      "logps/chosen": -1032.0,
      "logps/rejected": -1736.0,
      "loss": 0.0267,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.15625,
      "rewards/margins": 7.34375,
      "rewards/rejected": -14.5,
      "step": 2760
    },
    {
      "epoch": 2.8990057561486133,
      "grad_norm": 1.099532685024332,
      "learning_rate": 2.238194108195167e-05,
      "logits/chosen": -2.53125,
      "logits/rejected": 1.609375,
      "logps/chosen": -1080.0,
      "logps/rejected": -1776.0,
      "loss": 0.0311,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.21875,
      "rewards/margins": 7.5,
      "rewards/rejected": -14.75,
      "step": 2770
    },
    {
      "epoch": 2.909471480900052,
      "grad_norm": 1.1911395866069077,
      "learning_rate": 2.220023942342844e-05,
      "logits/chosen": -3.15625,
      "logits/rejected": 1.0703125,
      "logps/chosen": -1144.0,
      "logps/rejected": -1872.0,
      "loss": 0.0238,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.875,
      "rewards/margins": 7.75,
      "rewards/rejected": -15.625,
      "step": 2780
    },
    {
      "epoch": 2.9199372056514914,
      "grad_norm": 1.6577589564365567,
      "learning_rate": 2.2018687418790874e-05,
      "logits/chosen": -3.703125,
      "logits/rejected": 0.0810546875,
      "logps/chosen": -1064.0,
      "logps/rejected": -1864.0,
      "loss": 0.0235,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.3125,
      "rewards/margins": 8.3125,
      "rewards/rejected": -15.625,
      "step": 2790
    },
    {
      "epoch": 2.9304029304029307,
      "grad_norm": 0.8181355263028384,
      "learning_rate": 2.1837294772426948e-05,
      "logits/chosen": -2.9375,
      "logits/rejected": 0.5625,
      "logps/chosen": -1032.0,
      "logps/rejected": -1704.0,
      "loss": 0.0369,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.25,
      "rewards/margins": 7.28125,
      "rewards/rejected": -14.5,
      "step": 2800
    },
    {
      "epoch": 2.9304029304029307,
      "eval_logits/chosen": -1.453125,
      "eval_logits/rejected": 0.62890625,
      "eval_logps/chosen": -1272.0,
      "eval_logps/rejected": -1472.0,
      "eval_loss": 0.8085156083106995,
      "eval_rewards/accuracies": 0.72265625,
      "eval_rewards/chosen": -9.5625,
      "eval_rewards/margins": 2.3125,
      "eval_rewards/rejected": -11.875,
      "eval_runtime": 52.8836,
      "eval_samples_per_second": 37.819,
      "eval_steps_per_second": 0.605,
      "step": 2800
    },
    {
      "epoch": 2.9408686551543695,
      "grad_norm": 0.9435732480557755,
      "learning_rate": 2.1656071180206593e-05,
      "logits/chosen": -2.625,
      "logits/rejected": 1.0546875,
      "logps/chosen": -1120.0,
      "logps/rejected": -1952.0,
      "loss": 0.0278,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.90625,
      "rewards/margins": 8.5625,
      "rewards/rejected": -16.5,
      "step": 2810
    },
    {
      "epoch": 2.9513343799058083,
      "grad_norm": 1.2048992857915675,
      "learning_rate": 2.1475026328963376e-05,
      "logits/chosen": -2.953125,
      "logits/rejected": 1.390625,
      "logps/chosen": -1096.0,
      "logps/rejected": -1872.0,
      "loss": 0.0232,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.84375,
      "rewards/margins": 8.0,
      "rewards/rejected": -15.875,
      "step": 2820
    },
    {
      "epoch": 2.9618001046572475,
      "grad_norm": 1.5340965910243034,
      "learning_rate": 2.1294169895976746e-05,
      "logits/chosen": -3.078125,
      "logits/rejected": 0.890625,
      "logps/chosen": -1072.0,
      "logps/rejected": -1768.0,
      "loss": 0.0229,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.40625,
      "rewards/margins": 7.34375,
      "rewards/rejected": -14.75,
      "step": 2830
    },
    {
      "epoch": 2.9722658294086868,
      "grad_norm": 0.3265272820820435,
      "learning_rate": 2.111351154845472e-05,
      "logits/chosen": -1.546875,
      "logits/rejected": 2.71875,
      "logps/chosen": -1088.0,
      "logps/rejected": -1816.0,
      "loss": 0.0265,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.53125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -15.375,
      "step": 2840
    },
    {
      "epoch": 2.9827315541601256,
      "grad_norm": 0.5998260117426013,
      "learning_rate": 2.0933060943017215e-05,
      "logits/chosen": -1.9453125,
      "logits/rejected": 0.703125,
      "logps/chosen": -1020.0,
      "logps/rejected": -1784.0,
      "loss": 0.0278,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.0625,
      "rewards/margins": 7.75,
      "rewards/rejected": -14.8125,
      "step": 2850
    },
    {
      "epoch": 2.9931972789115644,
      "grad_norm": 0.54910635876851,
      "learning_rate": 2.0752827725179793e-05,
      "logits/chosen": -2.40625,
      "logits/rejected": 0.50390625,
      "logps/chosen": -1040.0,
      "logps/rejected": -1704.0,
      "loss": 0.0286,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.1875,
      "rewards/margins": 6.90625,
      "rewards/rejected": -14.0625,
      "step": 2860
    },
    {
      "epoch": 3.0036630036630036,
      "grad_norm": 0.146418403562972,
      "learning_rate": 2.0572821528838175e-05,
      "logits/chosen": -4.34375,
      "logits/rejected": 0.859375,
      "logps/chosen": -988.0,
      "logps/rejected": -1608.0,
      "loss": 0.019,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.375,
      "rewards/margins": 7.28125,
      "rewards/rejected": -13.6875,
      "step": 2870
    },
    {
      "epoch": 3.014128728414443,
      "grad_norm": 2.895012944047952,
      "learning_rate": 2.0393051975753196e-05,
      "logits/chosen": -3.90625,
      "logits/rejected": 0.2197265625,
      "logps/chosen": -860.0,
      "logps/rejected": -1616.0,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.625,
      "rewards/margins": 7.9375,
      "rewards/rejected": -13.5625,
      "step": 2880
    },
    {
      "epoch": 3.0245944531658817,
      "grad_norm": 0.1423538198589034,
      "learning_rate": 2.021352867503657e-05,
      "logits/chosen": -3.75,
      "logits/rejected": 1.3046875,
      "logps/chosen": -1008.0,
      "logps/rejected": -1824.0,
      "loss": 0.0114,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.75,
      "rewards/margins": 8.6875,
      "rewards/rejected": -15.5,
      "step": 2890
    },
    {
      "epoch": 3.035060177917321,
      "grad_norm": 0.2962179324568394,
      "learning_rate": 2.003426122263723e-05,
      "logits/chosen": -3.375,
      "logits/rejected": 0.69140625,
      "logps/chosen": -1032.0,
      "logps/rejected": -1896.0,
      "loss": 0.0154,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.09375,
      "rewards/margins": 8.9375,
      "rewards/rejected": -16.0,
      "step": 2900
    },
    {
      "epoch": 3.035060177917321,
      "eval_logits/chosen": -1.3125,
      "eval_logits/rejected": 0.9609375,
      "eval_logps/chosen": -1296.0,
      "eval_logps/rejected": -1520.0,
      "eval_loss": 0.8778906464576721,
      "eval_rewards/accuracies": 0.7265625,
      "eval_rewards/chosen": -9.875,
      "eval_rewards/margins": 2.5,
      "eval_rewards/rejected": -12.375,
      "eval_runtime": 52.8723,
      "eval_samples_per_second": 37.827,
      "eval_steps_per_second": 0.605,
      "step": 2900
    },
    {
      "epoch": 3.0455259026687598,
      "grad_norm": 0.03778973429382476,
      "learning_rate": 1.9855259200828412e-05,
      "logits/chosen": -2.96875,
      "logits/rejected": 1.5234375,
      "logps/chosen": -1080.0,
      "logps/rejected": -1992.0,
      "loss": 0.01,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.40625,
      "rewards/margins": 9.4375,
      "rewards/rejected": -16.75,
      "step": 2910
    },
    {
      "epoch": 3.055991627420199,
      "grad_norm": 0.05371775375140043,
      "learning_rate": 1.9676532177695442e-05,
      "logits/chosen": -2.78125,
      "logits/rejected": 1.546875,
      "logps/chosen": -1080.0,
      "logps/rejected": -2000.0,
      "loss": 0.0085,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.84375,
      "rewards/margins": 9.25,
      "rewards/rejected": -17.125,
      "step": 2920
    },
    {
      "epoch": 3.066457352171638,
      "grad_norm": 0.025662869341906102,
      "learning_rate": 1.9498089706624316e-05,
      "logits/chosen": -2.8125,
      "logits/rejected": 1.390625,
      "logps/chosen": -1112.0,
      "logps/rejected": -1976.0,
      "loss": 0.0112,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.84375,
      "rewards/margins": 9.0,
      "rewards/rejected": -16.75,
      "step": 2930
    },
    {
      "epoch": 3.076923076923077,
      "grad_norm": 0.04698152913607542,
      "learning_rate": 1.9319941325791043e-05,
      "logits/chosen": -3.140625,
      "logits/rejected": 1.9921875,
      "logps/chosen": -1112.0,
      "logps/rejected": -2032.0,
      "loss": 0.0136,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.75,
      "rewards/margins": 9.75,
      "rewards/rejected": -17.5,
      "step": 2940
    },
    {
      "epoch": 3.087388801674516,
      "grad_norm": 0.2216407872137013,
      "learning_rate": 1.9142096557651808e-05,
      "logits/chosen": -2.90625,
      "logits/rejected": 2.34375,
      "logps/chosen": -1120.0,
      "logps/rejected": -2008.0,
      "loss": 0.0076,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.875,
      "rewards/margins": 9.625,
      "rewards/rejected": -17.5,
      "step": 2950
    },
    {
      "epoch": 3.097854526425955,
      "grad_norm": 0.033181262564329926,
      "learning_rate": 1.8964564908433965e-05,
      "logits/chosen": -1.5546875,
      "logits/rejected": 2.046875,
      "logps/chosen": -1216.0,
      "logps/rejected": -2176.0,
      "loss": 0.008,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.6875,
      "rewards/margins": 9.875,
      "rewards/rejected": -18.625,
      "step": 2960
    },
    {
      "epoch": 3.108320251177394,
      "grad_norm": 0.034234981051473136,
      "learning_rate": 1.8787355867627933e-05,
      "logits/chosen": -1.5234375,
      "logits/rejected": 2.890625,
      "logps/chosen": -1104.0,
      "logps/rejected": -2176.0,
      "loss": 0.0053,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.125,
      "rewards/margins": 10.75,
      "rewards/rejected": -18.875,
      "step": 2970
    },
    {
      "epoch": 3.118785975928833,
      "grad_norm": 0.058508823478964314,
      "learning_rate": 1.8610478907479916e-05,
      "logits/chosen": -1.78125,
      "logits/rejected": 3.484375,
      "logps/chosen": -1240.0,
      "logps/rejected": -2400.0,
      "loss": 0.0057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.125,
      "rewards/margins": 11.8125,
      "rewards/rejected": -20.875,
      "step": 2980
    },
    {
      "epoch": 3.129251700680272,
      "grad_norm": 2.6421404482627393,
      "learning_rate": 1.8433943482485624e-05,
      "logits/chosen": -1.6015625,
      "logits/rejected": 1.6875,
      "logps/chosen": -1264.0,
      "logps/rejected": -2256.0,
      "loss": 0.0079,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.25,
      "rewards/margins": 10.0625,
      "rewards/rejected": -19.375,
      "step": 2990
    },
    {
      "epoch": 3.1397174254317113,
      "grad_norm": 0.272455042334525,
      "learning_rate": 1.8257759028884912e-05,
      "logits/chosen": -1.28125,
      "logits/rejected": 2.421875,
      "logps/chosen": -1200.0,
      "logps/rejected": -2288.0,
      "loss": 0.007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -19.75,
      "step": 3000
    },
    {
      "epoch": 3.1397174254317113,
      "eval_logits/chosen": 0.68359375,
      "eval_logits/rejected": 2.796875,
      "eval_logps/chosen": -1464.0,
      "eval_logps/rejected": -1728.0,
      "eval_loss": 0.9780234098434448,
      "eval_rewards/accuracies": 0.720703125,
      "eval_rewards/chosen": -11.5,
      "eval_rewards/margins": 2.875,
      "eval_rewards/rejected": -14.375,
      "eval_runtime": 52.8667,
      "eval_samples_per_second": 37.831,
      "eval_steps_per_second": 0.605,
      "step": 3000
    },
    {
      "epoch": 3.15018315018315,
      "grad_norm": 0.03150523012206682,
      "learning_rate": 1.8081934964157353e-05,
      "logits/chosen": -1.0234375,
      "logits/rejected": 3.34375,
      "logps/chosen": -1232.0,
      "logps/rejected": -2128.0,
      "loss": 0.0101,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.6875,
      "rewards/margins": 9.6875,
      "rewards/rejected": -18.375,
      "step": 3010
    },
    {
      "epoch": 3.1606488749345893,
      "grad_norm": 0.1206717172365664,
      "learning_rate": 1.7906480686518903e-05,
      "logits/chosen": -0.359375,
      "logits/rejected": 3.546875,
      "logps/chosen": -1184.0,
      "logps/rejected": -2112.0,
      "loss": 0.0083,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.5,
      "rewards/margins": 9.8125,
      "rewards/rejected": -18.25,
      "step": 3020
    },
    {
      "epoch": 3.171114599686028,
      "grad_norm": 0.033920179683535764,
      "learning_rate": 1.773140557441948e-05,
      "logits/chosen": -1.5078125,
      "logits/rejected": 4.125,
      "logps/chosen": -1152.0,
      "logps/rejected": -2064.0,
      "loss": 0.008,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.1875,
      "rewards/margins": 10.0625,
      "rewards/rejected": -18.25,
      "step": 3030
    },
    {
      "epoch": 3.1815803244374674,
      "grad_norm": 0.024285830587125388,
      "learning_rate": 1.7556718986041733e-05,
      "logits/chosen": -0.79296875,
      "logits/rejected": 3.734375,
      "logps/chosen": -1144.0,
      "logps/rejected": -2064.0,
      "loss": 0.0108,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.25,
      "rewards/margins": 9.6875,
      "rewards/rejected": -18.0,
      "step": 3040
    },
    {
      "epoch": 3.192046049188906,
      "grad_norm": 1.4925565821536846,
      "learning_rate": 1.7382430258800763e-05,
      "logits/chosen": -0.78125,
      "logits/rejected": 3.359375,
      "logps/chosen": -1112.0,
      "logps/rejected": -2128.0,
      "loss": 0.009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 10.375,
      "rewards/rejected": -18.625,
      "step": 3050
    },
    {
      "epoch": 3.2025117739403455,
      "grad_norm": 1.0235211261758,
      "learning_rate": 1.720854870884507e-05,
      "logits/chosen": 0.1494140625,
      "logits/rejected": 3.8125,
      "logps/chosen": -1208.0,
      "logps/rejected": -2192.0,
      "loss": 0.0152,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.0625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -19.125,
      "step": 3060
    },
    {
      "epoch": 3.2129774986917843,
      "grad_norm": 0.2888409223728506,
      "learning_rate": 1.7035083630558525e-05,
      "logits/chosen": -1.640625,
      "logits/rejected": 3.0625,
      "logps/chosen": -1152.0,
      "logps/rejected": -2032.0,
      "loss": 0.0048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 9.4375,
      "rewards/rejected": -17.5,
      "step": 3070
    },
    {
      "epoch": 3.2234432234432235,
      "grad_norm": 0.09681539714463304,
      "learning_rate": 1.6862044296063635e-05,
      "logits/chosen": -1.375,
      "logits/rejected": 2.46875,
      "logps/chosen": -1160.0,
      "logps/rejected": -2176.0,
      "loss": 0.006,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.625,
      "rewards/margins": 10.0,
      "rewards/rejected": -18.625,
      "step": 3080
    },
    {
      "epoch": 3.2339089481946623,
      "grad_norm": 0.611624799453792,
      "learning_rate": 1.6689439954725834e-05,
      "logits/chosen": -1.0234375,
      "logits/rejected": 2.546875,
      "logps/chosen": -1192.0,
      "logps/rejected": -2192.0,
      "loss": 0.0081,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.5625,
      "rewards/margins": 10.1875,
      "rewards/rejected": -18.75,
      "step": 3090
    },
    {
      "epoch": 3.2443746729461016,
      "grad_norm": 0.034054919296362146,
      "learning_rate": 1.651727983265918e-05,
      "logits/chosen": -1.9765625,
      "logits/rejected": 1.890625,
      "logps/chosen": -1192.0,
      "logps/rejected": -2096.0,
      "loss": 0.0059,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.1875,
      "rewards/margins": 9.6875,
      "rewards/rejected": -17.875,
      "step": 3100
    },
    {
      "epoch": 3.2443746729461016,
      "eval_logits/chosen": 0.0257568359375,
      "eval_logits/rejected": 2.21875,
      "eval_logps/chosen": -1424.0,
      "eval_logps/rejected": -1688.0,
      "eval_loss": 0.9792890548706055,
      "eval_rewards/accuracies": 0.708984375,
      "eval_rewards/chosen": -11.125,
      "eval_rewards/margins": 2.875,
      "eval_rewards/rejected": -14.0,
      "eval_runtime": 52.8828,
      "eval_samples_per_second": 37.819,
      "eval_steps_per_second": 0.605,
      "step": 3100
    },
    {
      "epoch": 3.2548403976975404,
      "grad_norm": 1.2632359318208952,
      "learning_rate": 1.6345573132233116e-05,
      "logits/chosen": -2.203125,
      "logits/rejected": 2.40625,
      "logps/chosen": -1144.0,
      "logps/rejected": -2096.0,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 9.9375,
      "rewards/rejected": -18.375,
      "step": 3110
    },
    {
      "epoch": 3.2653061224489797,
      "grad_norm": 0.17830708741981413,
      "learning_rate": 1.617432903158065e-05,
      "logits/chosen": -0.85546875,
      "logits/rejected": 2.734375,
      "logps/chosen": -1176.0,
      "logps/rejected": -2144.0,
      "loss": 0.0109,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.625,
      "rewards/margins": 9.9375,
      "rewards/rejected": -18.5,
      "step": 3120
    },
    {
      "epoch": 3.2757718472004185,
      "grad_norm": 0.015030610448317385,
      "learning_rate": 1.600355668410769e-05,
      "logits/chosen": -1.109375,
      "logits/rejected": 3.125,
      "logps/chosen": -1232.0,
      "logps/rejected": -2208.0,
      "loss": 0.0035,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.9375,
      "rewards/margins": 10.25,
      "rewards/rejected": -19.125,
      "step": 3130
    },
    {
      "epoch": 3.2862375719518577,
      "grad_norm": 0.27883925569127904,
      "learning_rate": 1.5833265218003852e-05,
      "logits/chosen": -0.90234375,
      "logits/rejected": 3.203125,
      "logps/chosen": -1208.0,
      "logps/rejected": -2240.0,
      "loss": 0.0115,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -8.9375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -19.5,
      "step": 3140
    },
    {
      "epoch": 3.2967032967032965,
      "grad_norm": 0.7036112100678287,
      "learning_rate": 1.566346373575445e-05,
      "logits/chosen": -0.443359375,
      "logits/rejected": 2.828125,
      "logps/chosen": -1192.0,
      "logps/rejected": -2208.0,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.0,
      "rewards/margins": 10.3125,
      "rewards/rejected": -19.375,
      "step": 3150
    },
    {
      "epoch": 3.307169021454736,
      "grad_norm": 0.036521152147152444,
      "learning_rate": 1.5494161313654032e-05,
      "logits/chosen": -0.7421875,
      "logits/rejected": 3.390625,
      "logps/chosen": -1224.0,
      "logps/rejected": -2160.0,
      "loss": 0.0088,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.875,
      "rewards/margins": 10.0,
      "rewards/rejected": -18.875,
      "step": 3160
    },
    {
      "epoch": 3.3176347462061746,
      "grad_norm": 0.21403998437602492,
      "learning_rate": 1.5325367001321162e-05,
      "logits/chosen": -1.3125,
      "logits/rejected": 3.46875,
      "logps/chosen": -1176.0,
      "logps/rejected": -2128.0,
      "loss": 0.0206,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -8.5,
      "rewards/margins": 10.125,
      "rewards/rejected": -18.625,
      "step": 3170
    },
    {
      "epoch": 3.328100470957614,
      "grad_norm": 0.4443822779711363,
      "learning_rate": 1.5157089821214745e-05,
      "logits/chosen": -0.78515625,
      "logits/rejected": 2.703125,
      "logps/chosen": -1136.0,
      "logps/rejected": -2064.0,
      "loss": 0.0102,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.0,
      "rewards/margins": 9.75,
      "rewards/rejected": -17.75,
      "step": 3180
    },
    {
      "epoch": 3.3385661957090527,
      "grad_norm": 0.08603805919540032,
      "learning_rate": 1.4989338768151714e-05,
      "logits/chosen": -0.78515625,
      "logits/rejected": 2.40625,
      "logps/chosen": -1112.0,
      "logps/rejected": -2064.0,
      "loss": 0.0091,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.90625,
      "rewards/margins": 9.6875,
      "rewards/rejected": -17.5,
      "step": 3190
    },
    {
      "epoch": 3.349031920460492,
      "grad_norm": 0.14729072438320479,
      "learning_rate": 1.482212280882626e-05,
      "logits/chosen": -0.734375,
      "logits/rejected": 2.3125,
      "logps/chosen": -1088.0,
      "logps/rejected": -2040.0,
      "loss": 0.0102,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.8125,
      "rewards/margins": 9.8125,
      "rewards/rejected": -17.625,
      "step": 3200
    },
    {
      "epoch": 3.349031920460492,
      "eval_logits/chosen": 0.75390625,
      "eval_logits/rejected": 2.765625,
      "eval_logps/chosen": -1424.0,
      "eval_logps/rejected": -1672.0,
      "eval_loss": 0.9822812676429749,
      "eval_rewards/accuracies": 0.71484375,
      "eval_rewards/chosen": -11.0625,
      "eval_rewards/margins": 2.828125,
      "eval_rewards/rejected": -13.875,
      "eval_runtime": 52.8816,
      "eval_samples_per_second": 37.82,
      "eval_steps_per_second": 0.605,
      "step": 3200
    },
    {
      "epoch": 3.359497645211931,
      "grad_norm": 0.08429283775876696,
      "learning_rate": 1.4655450881330557e-05,
      "logits/chosen": -1.078125,
      "logits/rejected": 2.90625,
      "logps/chosen": -1128.0,
      "logps/rejected": -2048.0,
      "loss": 0.0095,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.0625,
      "rewards/margins": 9.5625,
      "rewards/rejected": -17.625,
      "step": 3210
    },
    {
      "epoch": 3.36996336996337,
      "grad_norm": 0.20188899432806157,
      "learning_rate": 1.4489331894676955e-05,
      "logits/chosen": -0.765625,
      "logits/rejected": 3.015625,
      "logps/chosen": -1192.0,
      "logps/rejected": -2176.0,
      "loss": 0.0062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.8125,
      "rewards/margins": 10.0,
      "rewards/rejected": -18.75,
      "step": 3220
    },
    {
      "epoch": 3.380429094714809,
      "grad_norm": 0.4400190726675296,
      "learning_rate": 1.4323774728321799e-05,
      "logits/chosen": 0.0186767578125,
      "logits/rejected": 3.4375,
      "logps/chosen": -1216.0,
      "logps/rejected": -2256.0,
      "loss": 0.0121,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.0625,
      "rewards/margins": 10.6875,
      "rewards/rejected": -19.75,
      "step": 3230
    },
    {
      "epoch": 3.390894819466248,
      "grad_norm": 0.1780644502144959,
      "learning_rate": 1.4158788231690817e-05,
      "logits/chosen": -0.07080078125,
      "logits/rejected": 3.375,
      "logps/chosen": -1192.0,
      "logps/rejected": -2256.0,
      "loss": 0.0097,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.875,
      "rewards/margins": 10.875,
      "rewards/rejected": -19.75,
      "step": 3240
    },
    {
      "epoch": 3.4013605442176873,
      "grad_norm": 0.08336600862936035,
      "learning_rate": 1.399438122370606e-05,
      "logits/chosen": -0.302734375,
      "logits/rejected": 2.8125,
      "logps/chosen": -1136.0,
      "logps/rejected": -2080.0,
      "loss": 0.0069,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.125,
      "rewards/margins": 9.875,
      "rewards/rejected": -18.0,
      "step": 3250
    },
    {
      "epoch": 3.411826268969126,
      "grad_norm": 0.2844580711193598,
      "learning_rate": 1.383056249231453e-05,
      "logits/chosen": -0.296875,
      "logits/rejected": 2.546875,
      "logps/chosen": -1144.0,
      "logps/rejected": -2112.0,
      "loss": 0.0049,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.1875,
      "rewards/margins": 9.875,
      "rewards/rejected": -18.0,
      "step": 3260
    },
    {
      "epoch": 3.422291993720565,
      "grad_norm": 0.09143723360965565,
      "learning_rate": 1.366734079401844e-05,
      "logits/chosen": -0.171875,
      "logits/rejected": 3.078125,
      "logps/chosen": -1152.0,
      "logps/rejected": -2048.0,
      "loss": 0.005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 9.375,
      "rewards/rejected": -17.5,
      "step": 3270
    },
    {
      "epoch": 3.432757718472004,
      "grad_norm": 0.1015527005173294,
      "learning_rate": 1.3504724853407186e-05,
      "logits/chosen": -0.32421875,
      "logits/rejected": 2.6875,
      "logps/chosen": -1192.0,
      "logps/rejected": -2112.0,
      "loss": 0.008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 9.5625,
      "rewards/rejected": -18.125,
      "step": 3280
    },
    {
      "epoch": 3.4432234432234434,
      "grad_norm": 0.061434527839048896,
      "learning_rate": 1.3342723362690915e-05,
      "logits/chosen": -0.220703125,
      "logits/rejected": 2.75,
      "logps/chosen": -1176.0,
      "logps/rejected": -2176.0,
      "loss": 0.008,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.625,
      "rewards/margins": 10.25,
      "rewards/rejected": -18.875,
      "step": 3290
    },
    {
      "epoch": 3.4536891679748822,
      "grad_norm": 0.24663657756925686,
      "learning_rate": 1.3181344981236008e-05,
      "logits/chosen": 0.02490234375,
      "logits/rejected": 3.234375,
      "logps/chosen": -1168.0,
      "logps/rejected": -2224.0,
      "loss": 0.0082,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.5,
      "rewards/margins": 10.9375,
      "rewards/rejected": -19.375,
      "step": 3300
    },
    {
      "epoch": 3.4536891679748822,
      "eval_logits/chosen": 1.4453125,
      "eval_logits/rejected": 3.328125,
      "eval_logps/chosen": -1528.0,
      "eval_logps/rejected": -1800.0,
      "eval_loss": 1.0422968864440918,
      "eval_rewards/accuracies": 0.705078125,
      "eval_rewards/chosen": -12.1875,
      "eval_rewards/margins": 3.0,
      "eval_rewards/rejected": -15.1875,
      "eval_runtime": 52.8855,
      "eval_samples_per_second": 37.818,
      "eval_steps_per_second": 0.605,
      "step": 3300
    },
    {
      "epoch": 3.4641548927263215,
      "grad_norm": 0.07007637393090188,
      "learning_rate": 1.3020598335102141e-05,
      "logits/chosen": 0.453125,
      "logits/rejected": 3.671875,
      "logps/chosen": -1224.0,
      "logps/rejected": -2320.0,
      "loss": 0.0087,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.0625,
      "rewards/margins": 11.4375,
      "rewards/rejected": -20.5,
      "step": 3310
    },
    {
      "epoch": 3.4746206174777603,
      "grad_norm": 0.028916860560453693,
      "learning_rate": 1.2860492016581254e-05,
      "logits/chosen": -0.1982421875,
      "logits/rejected": 3.28125,
      "logps/chosen": -1168.0,
      "logps/rejected": -2176.0,
      "loss": 0.0159,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.75,
      "rewards/margins": 10.4375,
      "rewards/rejected": -19.25,
      "step": 3320
    },
    {
      "epoch": 3.4850863422291996,
      "grad_norm": 0.08385675983762503,
      "learning_rate": 1.2701034583738236e-05,
      "logits/chosen": -0.6484375,
      "logits/rejected": 3.0,
      "logps/chosen": -1192.0,
      "logps/rejected": -2176.0,
      "loss": 0.0048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 10.25,
      "rewards/rejected": -18.75,
      "step": 3330
    },
    {
      "epoch": 3.4955520669806384,
      "grad_norm": 0.03573166830757387,
      "learning_rate": 1.2542234559953481e-05,
      "logits/chosen": -0.55078125,
      "logits/rejected": 2.359375,
      "logps/chosen": -1120.0,
      "logps/rejected": -2192.0,
      "loss": 0.006,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.25,
      "rewards/margins": 10.75,
      "rewards/rejected": -19.0,
      "step": 3340
    },
    {
      "epoch": 3.506017791732077,
      "grad_norm": 0.2541379015084323,
      "learning_rate": 1.2384100433467292e-05,
      "logits/chosen": -0.498046875,
      "logits/rejected": 2.6875,
      "logps/chosen": -1128.0,
      "logps/rejected": -2112.0,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.125,
      "rewards/margins": 10.0625,
      "rewards/rejected": -18.125,
      "step": 3350
    },
    {
      "epoch": 3.5164835164835164,
      "grad_norm": 0.2175200998178627,
      "learning_rate": 1.2226640656926206e-05,
      "logits/chosen": -1.4296875,
      "logits/rejected": 2.65625,
      "logps/chosen": -1256.0,
      "logps/rejected": -2352.0,
      "loss": 0.0079,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.0625,
      "rewards/margins": 11.3125,
      "rewards/rejected": -20.375,
      "step": 3360
    },
    {
      "epoch": 3.5269492412349557,
      "grad_norm": 0.9318061276340431,
      "learning_rate": 1.2069863646931112e-05,
      "logits/chosen": -0.345703125,
      "logits/rejected": 3.46875,
      "logps/chosen": -1264.0,
      "logps/rejected": -2336.0,
      "loss": 0.0064,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.375,
      "rewards/margins": 11.0625,
      "rewards/rejected": -20.5,
      "step": 3370
    },
    {
      "epoch": 3.5374149659863945,
      "grad_norm": 1.028069173008785,
      "learning_rate": 1.1913777783587418e-05,
      "logits/chosen": -0.91796875,
      "logits/rejected": 3.40625,
      "logps/chosen": -1200.0,
      "logps/rejected": -2272.0,
      "loss": 0.0108,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.625,
      "rewards/margins": 11.25,
      "rewards/rejected": -19.875,
      "step": 3380
    },
    {
      "epoch": 3.5478806907378337,
      "grad_norm": 0.10508544826965832,
      "learning_rate": 1.1758391410057079e-05,
      "logits/chosen": -1.09375,
      "logits/rejected": 2.171875,
      "logps/chosen": -1152.0,
      "logps/rejected": -2192.0,
      "loss": 0.0066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 10.5625,
      "rewards/rejected": -18.875,
      "step": 3390
    },
    {
      "epoch": 3.5583464154892726,
      "grad_norm": 0.08953477427818697,
      "learning_rate": 1.1603712832112657e-05,
      "logits/chosen": -0.984375,
      "logits/rejected": 2.8125,
      "logps/chosen": -1184.0,
      "logps/rejected": -2192.0,
      "loss": 0.0109,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.75,
      "rewards/margins": 10.4375,
      "rewards/rejected": -19.125,
      "step": 3400
    },
    {
      "epoch": 3.5583464154892726,
      "eval_logits/chosen": 0.8671875,
      "eval_logits/rejected": 2.875,
      "eval_logps/chosen": -1456.0,
      "eval_logps/rejected": -1720.0,
      "eval_loss": 1.0224688053131104,
      "eval_rewards/accuracies": 0.716796875,
      "eval_rewards/chosen": -11.375,
      "eval_rewards/margins": 2.96875,
      "eval_rewards/rejected": -14.375,
      "eval_runtime": 52.9002,
      "eval_samples_per_second": 37.807,
      "eval_steps_per_second": 0.605,
      "step": 3400
    },
    {
      "epoch": 3.568812140240712,
      "grad_norm": 0.5504935902471659,
      "learning_rate": 1.1449750317693353e-05,
      "logits/chosen": -1.0078125,
      "logits/rejected": 3.015625,
      "logps/chosen": -1168.0,
      "logps/rejected": -2240.0,
      "loss": 0.0134,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.375,
      "rewards/margins": 11.0,
      "rewards/rejected": -19.375,
      "step": 3410
    },
    {
      "epoch": 3.5792778649921506,
      "grad_norm": 0.06561132538339516,
      "learning_rate": 1.1296512096463047e-05,
      "logits/chosen": -0.95703125,
      "logits/rejected": 2.796875,
      "logps/chosen": -1072.0,
      "logps/rejected": -2080.0,
      "loss": 0.0078,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.9375,
      "rewards/margins": 10.1875,
      "rewards/rejected": -18.125,
      "step": 3420
    },
    {
      "epoch": 3.58974358974359,
      "grad_norm": 0.3476124297679733,
      "learning_rate": 1.1144006359370453e-05,
      "logits/chosen": -1.2890625,
      "logits/rejected": 1.7734375,
      "logps/chosen": -1176.0,
      "logps/rejected": -2208.0,
      "loss": 0.01,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.5,
      "rewards/margins": 10.5625,
      "rewards/rejected": -19.0,
      "step": 3430
    },
    {
      "epoch": 3.6002093144950287,
      "grad_norm": 0.1391743907904667,
      "learning_rate": 1.0992241258211233e-05,
      "logits/chosen": -1.4765625,
      "logits/rejected": 2.421875,
      "logps/chosen": -1160.0,
      "logps/rejected": -2208.0,
      "loss": 0.0063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -18.875,
      "step": 3440
    },
    {
      "epoch": 3.610675039246468,
      "grad_norm": 0.05180346161416574,
      "learning_rate": 1.0841224905192294e-05,
      "logits/chosen": -1.3515625,
      "logits/rejected": 3.6875,
      "logps/chosen": -1112.0,
      "logps/rejected": -2080.0,
      "loss": 0.0049,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.90625,
      "rewards/margins": 10.25,
      "rewards/rejected": -18.25,
      "step": 3450
    },
    {
      "epoch": 3.6211407639979067,
      "grad_norm": 0.06659517905006358,
      "learning_rate": 1.069096537249816e-05,
      "logits/chosen": -1.1796875,
      "logits/rejected": 2.859375,
      "logps/chosen": -1064.0,
      "logps/rejected": -2080.0,
      "loss": 0.007,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.46875,
      "rewards/margins": 10.4375,
      "rewards/rejected": -17.875,
      "step": 3460
    },
    {
      "epoch": 3.631606488749346,
      "grad_norm": 0.2334302110267224,
      "learning_rate": 1.0541470691859532e-05,
      "logits/chosen": -0.7578125,
      "logits/rejected": 3.578125,
      "logps/chosen": -1200.0,
      "logps/rejected": -2208.0,
      "loss": 0.0117,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.5625,
      "rewards/margins": 10.5625,
      "rewards/rejected": -19.125,
      "step": 3470
    },
    {
      "epoch": 3.642072213500785,
      "grad_norm": 1.057650688155039,
      "learning_rate": 1.0392748854123913e-05,
      "logits/chosen": -0.76171875,
      "logits/rejected": 3.078125,
      "logps/chosen": -1104.0,
      "logps/rejected": -2096.0,
      "loss": 0.0117,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.71875,
      "rewards/margins": 10.4375,
      "rewards/rejected": -18.125,
      "step": 3480
    },
    {
      "epoch": 3.652537938252224,
      "grad_norm": 0.20649118172648792,
      "learning_rate": 1.0244807808828527e-05,
      "logits/chosen": -0.412109375,
      "logits/rejected": 3.765625,
      "logps/chosen": -1192.0,
      "logps/rejected": -2160.0,
      "loss": 0.007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 10.4375,
      "rewards/rejected": -19.0,
      "step": 3490
    },
    {
      "epoch": 3.663003663003663,
      "grad_norm": 0.04027065560164829,
      "learning_rate": 1.0097655463775347e-05,
      "logits/chosen": -0.9765625,
      "logits/rejected": 3.4375,
      "logps/chosen": -1168.0,
      "logps/rejected": -2112.0,
      "loss": 0.0098,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.25,
      "rewards/margins": 10.0,
      "rewards/rejected": -18.25,
      "step": 3500
    },
    {
      "epoch": 3.663003663003663,
      "eval_logits/chosen": 1.171875,
      "eval_logits/rejected": 3.1875,
      "eval_logps/chosen": -1456.0,
      "eval_logps/rejected": -1712.0,
      "eval_loss": 1.0070234537124634,
      "eval_rewards/accuracies": 0.7109375,
      "eval_rewards/chosen": -11.4375,
      "eval_rewards/margins": 2.84375,
      "eval_rewards/rejected": -14.25,
      "eval_runtime": 52.8862,
      "eval_samples_per_second": 37.817,
      "eval_steps_per_second": 0.605,
      "step": 3500
    },
    {
      "epoch": 3.673469387755102,
      "grad_norm": 0.3978887479848816,
      "learning_rate": 9.951299684608479e-06,
      "logits/chosen": -0.3984375,
      "logits/rejected": 3.53125,
      "logps/chosen": -1168.0,
      "logps/rejected": -2176.0,
      "loss": 0.0074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.6875,
      "rewards/margins": 10.3125,
      "rewards/rejected": -19.0,
      "step": 3510
    },
    {
      "epoch": 3.683935112506541,
      "grad_norm": 0.040155524578011734,
      "learning_rate": 9.805748294393658e-06,
      "logits/chosen": -0.10498046875,
      "logits/rejected": 3.484375,
      "logps/chosen": -1168.0,
      "logps/rejected": -2224.0,
      "loss": 0.0116,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.6875,
      "rewards/margins": 10.75,
      "rewards/rejected": -19.375,
      "step": 3520
    },
    {
      "epoch": 3.69440083725798,
      "grad_norm": 0.06256794091530021,
      "learning_rate": 9.661009073200105e-06,
      "logits/chosen": 0.6171875,
      "logits/rejected": 3.90625,
      "logps/chosen": -1304.0,
      "logps/rejected": -2288.0,
      "loss": 0.0092,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -9.5,
      "rewards/margins": 10.3125,
      "rewards/rejected": -19.75,
      "step": 3530
    },
    {
      "epoch": 3.704866562009419,
      "grad_norm": 0.13259944802218135,
      "learning_rate": 9.517089757684675e-06,
      "logits/chosen": -0.546875,
      "logits/rejected": 2.96875,
      "logps/chosen": -1200.0,
      "logps/rejected": -2256.0,
      "loss": 0.0098,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.8125,
      "rewards/margins": 10.9375,
      "rewards/rejected": -19.75,
      "step": 3540
    },
    {
      "epoch": 3.7153322867608582,
      "grad_norm": 0.282203368505097,
      "learning_rate": 9.373998040678317e-06,
      "logits/chosen": 0.2578125,
      "logits/rejected": 4.1875,
      "logps/chosen": -1152.0,
      "logps/rejected": -2192.0,
      "loss": 0.0081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.625,
      "rewards/margins": 10.6875,
      "rewards/rejected": -19.375,
      "step": 3550
    },
    {
      "epoch": 3.725798011512297,
      "grad_norm": 0.5695312875065408,
      "learning_rate": 9.231741570774851e-06,
      "logits/chosen": -0.3359375,
      "logits/rejected": 4.3125,
      "logps/chosen": -1240.0,
      "logps/rejected": -2320.0,
      "loss": 0.0062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.375,
      "rewards/margins": 11.3125,
      "rewards/rejected": -20.625,
      "step": 3560
    },
    {
      "epoch": 3.7362637362637363,
      "grad_norm": 0.01268718846210354,
      "learning_rate": 9.090327951922147e-06,
      "logits/chosen": -0.12890625,
      "logits/rejected": 3.140625,
      "logps/chosen": -1240.0,
      "logps/rejected": -2464.0,
      "loss": 0.0069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.4375,
      "rewards/margins": 12.3125,
      "rewards/rejected": -21.75,
      "step": 3570
    },
    {
      "epoch": 3.7467294610151756,
      "grad_norm": 0.16298241858505716,
      "learning_rate": 8.949764743015668e-06,
      "logits/chosen": -0.41796875,
      "logits/rejected": 2.421875,
      "logps/chosen": -1216.0,
      "logps/rejected": -2320.0,
      "loss": 0.0106,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.75,
      "rewards/margins": 11.3125,
      "rewards/rejected": -20.125,
      "step": 3580
    },
    {
      "epoch": 3.7571951857666144,
      "grad_norm": 0.21812828668959902,
      "learning_rate": 8.810059457494474e-06,
      "logits/chosen": -0.466796875,
      "logits/rejected": 3.5625,
      "logps/chosen": -1248.0,
      "logps/rejected": -2304.0,
      "loss": 0.0065,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.1875,
      "rewards/margins": 10.8125,
      "rewards/rejected": -20.0,
      "step": 3590
    },
    {
      "epoch": 3.767660910518053,
      "grad_norm": 0.256677383911918,
      "learning_rate": 8.671219562939514e-06,
      "logits/chosen": -0.69921875,
      "logits/rejected": 3.234375,
      "logps/chosen": -1192.0,
      "logps/rejected": -2256.0,
      "loss": 0.007,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -9.0,
      "rewards/margins": 10.8125,
      "rewards/rejected": -19.875,
      "step": 3600
    },
    {
      "epoch": 3.767660910518053,
      "eval_logits/chosen": 0.80859375,
      "eval_logits/rejected": 2.859375,
      "eval_logps/chosen": -1512.0,
      "eval_logps/rejected": -1776.0,
      "eval_loss": 1.0389765501022339,
      "eval_rewards/accuracies": 0.71484375,
      "eval_rewards/chosen": -11.9375,
      "eval_rewards/margins": 3.0,
      "eval_rewards/rejected": -14.9375,
      "eval_runtime": 52.8868,
      "eval_samples_per_second": 37.817,
      "eval_steps_per_second": 0.605,
      "step": 3600
    },
    {
      "epoch": 3.7781266352694924,
      "grad_norm": 0.3973905986468847,
      "learning_rate": 8.533252480674552e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": 2.84375,
      "logps/chosen": -1232.0,
      "logps/rejected": -2224.0,
      "loss": 0.013,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.125,
      "rewards/margins": 10.1875,
      "rewards/rejected": -19.25,
      "step": 3610
    },
    {
      "epoch": 3.7885923600209317,
      "grad_norm": 0.017282421705160138,
      "learning_rate": 8.396165585369473e-06,
      "logits/chosen": -0.9296875,
      "logits/rejected": 3.1875,
      "logps/chosen": -1248.0,
      "logps/rejected": -2256.0,
      "loss": 0.0053,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.875,
      "rewards/margins": 10.6875,
      "rewards/rejected": -19.625,
      "step": 3620
    },
    {
      "epoch": 3.7990580847723705,
      "grad_norm": 0.06936587201416461,
      "learning_rate": 8.259966204646036e-06,
      "logits/chosen": -0.78515625,
      "logits/rejected": 3.34375,
      "logps/chosen": -1152.0,
      "logps/rejected": -2272.0,
      "loss": 0.0076,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.125,
      "rewards/margins": 11.625,
      "rewards/rejected": -19.75,
      "step": 3630
    },
    {
      "epoch": 3.8095238095238093,
      "grad_norm": 0.14962645684813214,
      "learning_rate": 8.124661618686241e-06,
      "logits/chosen": -0.4296875,
      "logits/rejected": 3.015625,
      "logps/chosen": -1176.0,
      "logps/rejected": -2160.0,
      "loss": 0.0151,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -8.625,
      "rewards/margins": 10.125,
      "rewards/rejected": -18.75,
      "step": 3640
    },
    {
      "epoch": 3.8199895342752486,
      "grad_norm": 0.6158368813659576,
      "learning_rate": 7.99025905984316e-06,
      "logits/chosen": -0.1220703125,
      "logits/rejected": 3.234375,
      "logps/chosen": -1192.0,
      "logps/rejected": -2240.0,
      "loss": 0.0104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.9375,
      "rewards/margins": 10.625,
      "rewards/rejected": -19.5,
      "step": 3650
    },
    {
      "epoch": 3.830455259026688,
      "grad_norm": 0.047097837629874094,
      "learning_rate": 7.856765712254393e-06,
      "logits/chosen": 0.2431640625,
      "logits/rejected": 3.875,
      "logps/chosen": -1288.0,
      "logps/rejected": -2256.0,
      "loss": 0.0066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.25,
      "rewards/margins": 10.375,
      "rewards/rejected": -19.625,
      "step": 3660
    },
    {
      "epoch": 3.8409209837781266,
      "grad_norm": 0.025503821495670966,
      "learning_rate": 7.724188711457997e-06,
      "logits/chosen": -0.5546875,
      "logits/rejected": 2.84375,
      "logps/chosen": -1224.0,
      "logps/rejected": -2304.0,
      "loss": 0.0082,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.9375,
      "rewards/margins": 11.0625,
      "rewards/rejected": -20.0,
      "step": 3670
    },
    {
      "epoch": 3.8513867085295654,
      "grad_norm": 0.09958802810357707,
      "learning_rate": 7.5925351440111175e-06,
      "logits/chosen": -0.5703125,
      "logits/rejected": 3.96875,
      "logps/chosen": -1256.0,
      "logps/rejected": -2320.0,
      "loss": 0.0051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.1875,
      "rewards/margins": 11.1875,
      "rewards/rejected": -20.375,
      "step": 3680
    },
    {
      "epoch": 3.8618524332810047,
      "grad_norm": 0.34872245324217366,
      "learning_rate": 7.461812047111172e-06,
      "logits/chosen": -0.314453125,
      "logits/rejected": 3.890625,
      "logps/chosen": -1296.0,
      "logps/rejected": -2288.0,
      "loss": 0.0052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.5,
      "rewards/margins": 10.6875,
      "rewards/rejected": -20.125,
      "step": 3690
    },
    {
      "epoch": 3.872318158032444,
      "grad_norm": 0.1344362341791477,
      "learning_rate": 7.332026408219742e-06,
      "logits/chosen": -0.2373046875,
      "logits/rejected": 4.03125,
      "logps/chosen": -1288.0,
      "logps/rejected": -2336.0,
      "loss": 0.0057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.625,
      "rewards/margins": 11.125,
      "rewards/rejected": -20.75,
      "step": 3700
    },
    {
      "epoch": 3.872318158032444,
      "eval_logits/chosen": 1.5,
      "eval_logits/rejected": 3.453125,
      "eval_logps/chosen": -1584.0,
      "eval_logps/rejected": -1864.0,
      "eval_loss": 1.0701953172683716,
      "eval_rewards/accuracies": 0.703125,
      "eval_rewards/chosen": -12.75,
      "eval_rewards/margins": 3.0625,
      "eval_rewards/rejected": -15.8125,
      "eval_runtime": 52.8941,
      "eval_samples_per_second": 37.811,
      "eval_steps_per_second": 0.605,
      "step": 3700
    },
    {
      "epoch": 3.8827838827838828,
      "grad_norm": 0.020992160364998343,
      "learning_rate": 7.203185164688992e-06,
      "logits/chosen": 0.263671875,
      "logits/rejected": 3.671875,
      "logps/chosen": -1272.0,
      "logps/rejected": -2464.0,
      "loss": 0.0035,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.8125,
      "rewards/margins": 12.0,
      "rewards/rejected": -21.875,
      "step": 3710
    },
    {
      "epoch": 3.8932496075353216,
      "grad_norm": 0.07356976842643481,
      "learning_rate": 7.075295203390931e-06,
      "logits/chosen": -0.1162109375,
      "logits/rejected": 3.015625,
      "logps/chosen": -1248.0,
      "logps/rejected": -2320.0,
      "loss": 0.0078,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.375,
      "rewards/margins": 11.0,
      "rewards/rejected": -20.375,
      "step": 3720
    },
    {
      "epoch": 3.903715332286761,
      "grad_norm": 0.203253833608076,
      "learning_rate": 6.948363360349247e-06,
      "logits/chosen": -1.078125,
      "logits/rejected": 3.578125,
      "logps/chosen": -1280.0,
      "logps/rejected": -2336.0,
      "loss": 0.0058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.5625,
      "rewards/margins": 11.0,
      "rewards/rejected": -20.5,
      "step": 3730
    },
    {
      "epoch": 3.9141810570382,
      "grad_norm": 0.31356668119037906,
      "learning_rate": 6.822396420373942e-06,
      "logits/chosen": -0.380859375,
      "logits/rejected": 2.90625,
      "logps/chosen": -1240.0,
      "logps/rejected": -2352.0,
      "loss": 0.0085,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.0625,
      "rewards/margins": 11.1875,
      "rewards/rejected": -20.25,
      "step": 3740
    },
    {
      "epoch": 3.924646781789639,
      "grad_norm": 0.07165119195640822,
      "learning_rate": 6.697401116698626e-06,
      "logits/chosen": -1.015625,
      "logits/rejected": 3.03125,
      "logps/chosen": -1224.0,
      "logps/rejected": -2320.0,
      "loss": 0.0065,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.125,
      "rewards/margins": 11.25,
      "rewards/rejected": -20.375,
      "step": 3750
    },
    {
      "epoch": 3.9351125065410777,
      "grad_norm": 0.08827057592402958,
      "learning_rate": 6.573384130620638e-06,
      "logits/chosen": 0.9453125,
      "logits/rejected": 3.609375,
      "logps/chosen": -1248.0,
      "logps/rejected": -2368.0,
      "loss": 0.0094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.375,
      "rewards/margins": 11.4375,
      "rewards/rejected": -20.875,
      "step": 3760
    },
    {
      "epoch": 3.945578231292517,
      "grad_norm": 0.048809836235047845,
      "learning_rate": 6.450352091143891e-06,
      "logits/chosen": -0.443359375,
      "logits/rejected": 4.21875,
      "logps/chosen": -1256.0,
      "logps/rejected": -2304.0,
      "loss": 0.0097,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.25,
      "rewards/margins": 11.0,
      "rewards/rejected": -20.25,
      "step": 3770
    },
    {
      "epoch": 3.956043956043956,
      "grad_norm": 0.019544562581198487,
      "learning_rate": 6.328311574624582e-06,
      "logits/chosen": -0.671875,
      "logits/rejected": 2.90625,
      "logps/chosen": -1272.0,
      "logps/rejected": -2352.0,
      "loss": 0.0059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.4375,
      "rewards/margins": 10.875,
      "rewards/rejected": -20.375,
      "step": 3780
    },
    {
      "epoch": 3.966509680795395,
      "grad_norm": 0.13614961147873206,
      "learning_rate": 6.20726910441961e-06,
      "logits/chosen": 0.09619140625,
      "logits/rejected": 3.078125,
      "logps/chosen": -1208.0,
      "logps/rejected": -2256.0,
      "loss": 0.0061,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.0625,
      "rewards/margins": 10.625,
      "rewards/rejected": -19.625,
      "step": 3790
    },
    {
      "epoch": 3.9769754055468343,
      "grad_norm": 0.10878858635322676,
      "learning_rate": 6.087231150537933e-06,
      "logits/chosen": -0.041015625,
      "logits/rejected": 3.71875,
      "logps/chosen": -1248.0,
      "logps/rejected": -2288.0,
      "loss": 0.0054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.0625,
      "rewards/margins": 11.25,
      "rewards/rejected": -20.25,
      "step": 3800
    },
    {
      "epoch": 3.9769754055468343,
      "eval_logits/chosen": 1.46875,
      "eval_logits/rejected": 3.40625,
      "eval_logps/chosen": -1560.0,
      "eval_logps/rejected": -1832.0,
      "eval_loss": 1.0484999418258667,
      "eval_rewards/accuracies": 0.703125,
      "eval_rewards/chosen": -12.4375,
      "eval_rewards/margins": 3.0,
      "eval_rewards/rejected": -15.4375,
      "eval_runtime": 52.8974,
      "eval_samples_per_second": 37.809,
      "eval_steps_per_second": 0.605,
      "step": 3800
    },
    {
      "epoch": 3.987441130298273,
      "grad_norm": 2.2928026898675373,
      "learning_rate": 5.968204129294705e-06,
      "logits/chosen": -0.140625,
      "logits/rejected": 3.0625,
      "logps/chosen": -1232.0,
      "logps/rejected": -2288.0,
      "loss": 0.0063,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.125,
      "rewards/margins": 10.875,
      "rewards/rejected": -20.0,
      "step": 3810
    },
    {
      "epoch": 3.9979068550497123,
      "grad_norm": 0.08119257378657181,
      "learning_rate": 5.850194402968317e-06,
      "logits/chosen": -0.76953125,
      "logits/rejected": 3.109375,
      "logps/chosen": -1216.0,
      "logps/rejected": -2288.0,
      "loss": 0.006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.0,
      "rewards/margins": 10.9375,
      "rewards/rejected": -20.0,
      "step": 3820
    },
    {
      "epoch": 4.008372579801152,
      "grad_norm": 0.014981417991040193,
      "learning_rate": 5.733208279460331e-06,
      "logits/chosen": -0.47265625,
      "logits/rejected": 3.59375,
      "logps/chosen": -1248.0,
      "logps/rejected": -2304.0,
      "loss": 0.0037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.0625,
      "rewards/margins": 10.9375,
      "rewards/rejected": -20.0,
      "step": 3830
    },
    {
      "epoch": 4.01883830455259,
      "grad_norm": 0.02009579096970186,
      "learning_rate": 5.617252011958268e-06,
      "logits/chosen": -0.2734375,
      "logits/rejected": 2.921875,
      "logps/chosen": -1224.0,
      "logps/rejected": -2352.0,
      "loss": 0.0053,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.0,
      "rewards/margins": 11.3125,
      "rewards/rejected": -20.25,
      "step": 3840
    },
    {
      "epoch": 4.029304029304029,
      "grad_norm": 0.03084951999687856,
      "learning_rate": 5.502331798601429e-06,
      "logits/chosen": -0.87109375,
      "logits/rejected": 3.0625,
      "logps/chosen": -1224.0,
      "logps/rejected": -2240.0,
      "loss": 0.005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.0,
      "rewards/margins": 10.5625,
      "rewards/rejected": -19.625,
      "step": 3850
    },
    {
      "epoch": 4.0397697540554685,
      "grad_norm": 0.015498781392122992,
      "learning_rate": 5.388453782149514e-06,
      "logits/chosen": -0.03173828125,
      "logits/rejected": 3.171875,
      "logps/chosen": -1304.0,
      "logps/rejected": -2368.0,
      "loss": 0.0059,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.4375,
      "rewards/margins": 11.0,
      "rewards/rejected": -20.5,
      "step": 3860
    },
    {
      "epoch": 4.050235478806908,
      "grad_norm": 0.02232794620336459,
      "learning_rate": 5.275624049654318e-06,
      "logits/chosen": 0.3125,
      "logits/rejected": 3.578125,
      "logps/chosen": -1248.0,
      "logps/rejected": -2448.0,
      "loss": 0.0059,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.375,
      "rewards/margins": 12.375,
      "rewards/rejected": -21.75,
      "step": 3870
    },
    {
      "epoch": 4.060701203558346,
      "grad_norm": 0.020459293734781368,
      "learning_rate": 5.163848632134358e-06,
      "logits/chosen": -0.5703125,
      "logits/rejected": 3.40625,
      "logps/chosen": -1288.0,
      "logps/rejected": -2480.0,
      "loss": 0.007,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.5625,
      "rewards/margins": 12.1875,
      "rewards/rejected": -21.75,
      "step": 3880
    },
    {
      "epoch": 4.071166928309785,
      "grad_norm": 0.01373231281615159,
      "learning_rate": 5.053133504252511e-06,
      "logits/chosen": -0.49609375,
      "logits/rejected": 3.6875,
      "logps/chosen": -1296.0,
      "logps/rejected": -2400.0,
      "loss": 0.0095,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.5,
      "rewards/margins": 11.6875,
      "rewards/rejected": -21.125,
      "step": 3890
    },
    {
      "epoch": 4.081632653061225,
      "grad_norm": 0.08307094209547573,
      "learning_rate": 4.94348458399663e-06,
      "logits/chosen": -0.47265625,
      "logits/rejected": 3.1875,
      "logps/chosen": -1168.0,
      "logps/rejected": -2288.0,
      "loss": 0.0037,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.875,
      "rewards/margins": 11.375,
      "rewards/rejected": -20.25,
      "step": 3900
    },
    {
      "epoch": 4.081632653061225,
      "eval_logits/chosen": 1.6171875,
      "eval_logits/rejected": 3.546875,
      "eval_logps/chosen": -1600.0,
      "eval_logps/rejected": -1880.0,
      "eval_loss": 1.0904687643051147,
      "eval_rewards/accuracies": 0.703125,
      "eval_rewards/chosen": -12.8125,
      "eval_rewards/margins": 3.140625,
      "eval_rewards/rejected": -15.9375,
      "eval_runtime": 52.877,
      "eval_samples_per_second": 37.824,
      "eval_steps_per_second": 0.605,
      "step": 3900
    },
    {
      "epoch": 4.092098377812664,
      "grad_norm": 0.03337438011237956,
      "learning_rate": 4.834907732363217e-06,
      "logits/chosen": 7.343292236328125e-05,
      "logits/rejected": 3.859375,
      "logps/chosen": -1248.0,
      "logps/rejected": -2352.0,
      "loss": 0.0059,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.4375,
      "rewards/margins": 11.5,
      "rewards/rejected": -21.0,
      "step": 3910
    },
    {
      "epoch": 4.102564102564102,
      "grad_norm": 0.019973325709137745,
      "learning_rate": 4.727408753044155e-06,
      "logits/chosen": 0.248046875,
      "logits/rejected": 3.765625,
      "logps/chosen": -1352.0,
      "logps/rejected": -2464.0,
      "loss": 0.0067,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -10.0,
      "rewards/margins": 11.75,
      "rewards/rejected": -21.75,
      "step": 3920
    },
    {
      "epoch": 4.1130298273155415,
      "grad_norm": 0.3108228360137749,
      "learning_rate": 4.620993392116493e-06,
      "logits/chosen": -0.109375,
      "logits/rejected": 3.6875,
      "logps/chosen": -1288.0,
      "logps/rejected": -2416.0,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.75,
      "rewards/margins": 11.375,
      "rewards/rejected": -21.125,
      "step": 3930
    },
    {
      "epoch": 4.123495552066981,
      "grad_norm": 0.013055671038019035,
      "learning_rate": 4.515667337735277e-06,
      "logits/chosen": -0.640625,
      "logits/rejected": 3.921875,
      "logps/chosen": -1288.0,
      "logps/rejected": -2416.0,
      "loss": 0.0058,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.4375,
      "rewards/margins": 11.75,
      "rewards/rejected": -21.25,
      "step": 3940
    },
    {
      "epoch": 4.13396127681842,
      "grad_norm": 0.014303222620106174,
      "learning_rate": 4.4114362198295155e-06,
      "logits/chosen": -0.251953125,
      "logits/rejected": 3.65625,
      "logps/chosen": -1328.0,
      "logps/rejected": -2480.0,
      "loss": 0.009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.9375,
      "rewards/margins": 12.0,
      "rewards/rejected": -21.875,
      "step": 3950
    },
    {
      "epoch": 4.144427001569858,
      "grad_norm": 0.0427935909815543,
      "learning_rate": 4.308305609801244e-06,
      "logits/chosen": -0.73046875,
      "logits/rejected": 3.390625,
      "logps/chosen": -1272.0,
      "logps/rejected": -2320.0,
      "loss": 0.0054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.5625,
      "rewards/margins": 10.9375,
      "rewards/rejected": -20.5,
      "step": 3960
    },
    {
      "epoch": 4.154892726321298,
      "grad_norm": 0.025518900183695518,
      "learning_rate": 4.206281020227754e-06,
      "logits/chosen": -0.08056640625,
      "logits/rejected": 3.65625,
      "logps/chosen": -1344.0,
      "logps/rejected": -2432.0,
      "loss": 0.0047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.0,
      "rewards/margins": 11.3125,
      "rewards/rejected": -21.25,
      "step": 3970
    },
    {
      "epoch": 4.165358451072737,
      "grad_norm": 0.010020871741228683,
      "learning_rate": 4.105367904566873e-06,
      "logits/chosen": 0.8125,
      "logits/rejected": 3.953125,
      "logps/chosen": -1288.0,
      "logps/rejected": -2464.0,
      "loss": 0.0037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.625,
      "rewards/margins": 11.875,
      "rewards/rejected": -21.5,
      "step": 3980
    },
    {
      "epoch": 4.175824175824176,
      "grad_norm": 0.011681294736082001,
      "learning_rate": 4.005571656865503e-06,
      "logits/chosen": -0.0732421875,
      "logits/rejected": 4.03125,
      "logps/chosen": -1320.0,
      "logps/rejected": -2432.0,
      "loss": 0.0069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.8125,
      "rewards/margins": 11.5625,
      "rewards/rejected": -21.375,
      "step": 3990
    },
    {
      "epoch": 4.1862899005756145,
      "grad_norm": 0.26505203952414647,
      "learning_rate": 3.906897611471291e-06,
      "logits/chosen": 0.1298828125,
      "logits/rejected": 3.890625,
      "logps/chosen": -1320.0,
      "logps/rejected": -2528.0,
      "loss": 0.0031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.9375,
      "rewards/margins": 12.5625,
      "rewards/rejected": -22.5,
      "step": 4000
    },
    {
      "epoch": 4.1862899005756145,
      "eval_logits/chosen": 1.65625,
      "eval_logits/rejected": 3.609375,
      "eval_logps/chosen": -1616.0,
      "eval_logps/rejected": -1912.0,
      "eval_loss": 1.1163359880447388,
      "eval_rewards/accuracies": 0.701171875,
      "eval_rewards/chosen": -13.0625,
      "eval_rewards/margins": 3.21875,
      "eval_rewards/rejected": -16.25,
      "eval_runtime": 52.8881,
      "eval_samples_per_second": 37.816,
      "eval_steps_per_second": 0.605,
      "step": 4000
    },
    {
      "epoch": 4.196755625327054,
      "grad_norm": 0.06020278446533552,
      "learning_rate": 3.8093510427475027e-06,
      "logits/chosen": 0.47265625,
      "logits/rejected": 3.859375,
      "logps/chosen": -1280.0,
      "logps/rejected": -2448.0,
      "loss": 0.0059,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.625,
      "rewards/margins": 11.8125,
      "rewards/rejected": -21.5,
      "step": 4010
    },
    {
      "epoch": 4.207221350078493,
      "grad_norm": 0.013675313695423432,
      "learning_rate": 3.712937164791061e-06,
      "logits/chosen": 0.1328125,
      "logits/rejected": 2.875,
      "logps/chosen": -1280.0,
      "logps/rejected": -2464.0,
      "loss": 0.0068,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.6875,
      "rewards/margins": 11.8125,
      "rewards/rejected": -21.5,
      "step": 4020
    },
    {
      "epoch": 4.217687074829932,
      "grad_norm": 0.015820780338319115,
      "learning_rate": 3.617661131153871e-06,
      "logits/chosen": -0.984375,
      "logits/rejected": 3.09375,
      "logps/chosen": -1312.0,
      "logps/rejected": -2384.0,
      "loss": 0.0057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.625,
      "rewards/margins": 11.25,
      "rewards/rejected": -20.875,
      "step": 4030
    },
    {
      "epoch": 4.228152799581371,
      "grad_norm": 1.2290489445593125,
      "learning_rate": 3.523528034567361e-06,
      "logits/chosen": 0.0400390625,
      "logits/rejected": 3.71875,
      "logps/chosen": -1360.0,
      "logps/rejected": -2528.0,
      "loss": 0.0071,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.1875,
      "rewards/margins": 12.0,
      "rewards/rejected": -22.125,
      "step": 4040
    },
    {
      "epoch": 4.23861852433281,
      "grad_norm": 0.014275682691672148,
      "learning_rate": 3.4305429066702284e-06,
      "logits/chosen": 0.0120849609375,
      "logits/rejected": 3.40625,
      "logps/chosen": -1392.0,
      "logps/rejected": -2560.0,
      "loss": 0.0025,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.375,
      "rewards/margins": 12.0,
      "rewards/rejected": -22.375,
      "step": 4050
    },
    {
      "epoch": 4.249084249084249,
      "grad_norm": 0.01693962235310755,
      "learning_rate": 3.3387107177395044e-06,
      "logits/chosen": -0.23046875,
      "logits/rejected": 3.578125,
      "logps/chosen": -1304.0,
      "logps/rejected": -2416.0,
      "loss": 0.0049,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.875,
      "rewards/margins": 11.5,
      "rewards/rejected": -21.375,
      "step": 4060
    },
    {
      "epoch": 4.259549973835688,
      "grad_norm": 0.015628704589392063,
      "learning_rate": 3.2480363764248843e-06,
      "logits/chosen": -0.61328125,
      "logits/rejected": 3.390625,
      "logps/chosen": -1296.0,
      "logps/rejected": -2432.0,
      "loss": 0.0075,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.75,
      "rewards/margins": 11.625,
      "rewards/rejected": -21.375,
      "step": 4070
    },
    {
      "epoch": 4.270015698587127,
      "grad_norm": 0.023587527509885093,
      "learning_rate": 3.158524729486351e-06,
      "logits/chosen": -0.43359375,
      "logits/rejected": 3.203125,
      "logps/chosen": -1264.0,
      "logps/rejected": -2336.0,
      "loss": 0.0042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.3125,
      "rewards/margins": 11.0625,
      "rewards/rejected": -20.375,
      "step": 4080
    },
    {
      "epoch": 4.280481423338566,
      "grad_norm": 0.05923026570392513,
      "learning_rate": 3.070180561535088e-06,
      "logits/chosen": 0.33203125,
      "logits/rejected": 4.0,
      "logps/chosen": -1344.0,
      "logps/rejected": -2496.0,
      "loss": 0.0025,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.125,
      "rewards/margins": 11.8125,
      "rewards/rejected": -22.0,
      "step": 4090
    },
    {
      "epoch": 4.290947148090005,
      "grad_norm": 0.017270659248406598,
      "learning_rate": 2.983008594777745e-06,
      "logits/chosen": -0.1416015625,
      "logits/rejected": 3.296875,
      "logps/chosen": -1344.0,
      "logps/rejected": -2544.0,
      "loss": 0.0037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.75,
      "rewards/margins": 12.25,
      "rewards/rejected": -22.0,
      "step": 4100
    },
    {
      "epoch": 4.290947148090005,
      "eval_logits/chosen": 1.65625,
      "eval_logits/rejected": 3.609375,
      "eval_logps/chosen": -1624.0,
      "eval_logps/rejected": -1920.0,
      "eval_loss": 1.1256483793258667,
      "eval_rewards/accuracies": 0.708984375,
      "eval_rewards/chosen": -13.125,
      "eval_rewards/margins": 3.265625,
      "eval_rewards/rejected": -16.375,
      "eval_runtime": 52.8983,
      "eval_samples_per_second": 37.808,
      "eval_steps_per_second": 0.605,
      "step": 4100
    },
    {
      "epoch": 4.3014128728414445,
      "grad_norm": 0.03873780177102751,
      "learning_rate": 2.8970134887640137e-06,
      "logits/chosen": -0.19140625,
      "logits/rejected": 3.5,
      "logps/chosen": -1304.0,
      "logps/rejected": -2432.0,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.5625,
      "rewards/margins": 11.875,
      "rewards/rejected": -21.5,
      "step": 4110
    },
    {
      "epoch": 4.311878597592884,
      "grad_norm": 0.01075555055581885,
      "learning_rate": 2.8121998401375804e-06,
      "logits/chosen": 0.1796875,
      "logits/rejected": 3.84375,
      "logps/chosen": -1304.0,
      "logps/rejected": -2432.0,
      "loss": 0.0108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.8125,
      "rewards/margins": 11.8125,
      "rewards/rejected": -21.625,
      "step": 4120
    },
    {
      "epoch": 4.322344322344322,
      "grad_norm": 0.014744483880713274,
      "learning_rate": 2.7285721823904044e-06,
      "logits/chosen": 0.0302734375,
      "logits/rejected": 3.453125,
      "logps/chosen": -1296.0,
      "logps/rejected": -2512.0,
      "loss": 0.0061,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.875,
      "rewards/margins": 12.1875,
      "rewards/rejected": -22.0,
      "step": 4130
    },
    {
      "epoch": 4.332810047095761,
      "grad_norm": 0.050764409494126105,
      "learning_rate": 2.6461349856204077e-06,
      "logits/chosen": 0.1064453125,
      "logits/rejected": 3.125,
      "logps/chosen": -1240.0,
      "logps/rejected": -2480.0,
      "loss": 0.008,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.5,
      "rewards/margins": 12.375,
      "rewards/rejected": -21.875,
      "step": 4140
    },
    {
      "epoch": 4.343275771847201,
      "grad_norm": 0.12487661904822162,
      "learning_rate": 2.5648926562925222e-06,
      "logits/chosen": 0.169921875,
      "logits/rejected": 4.3125,
      "logps/chosen": -1288.0,
      "logps/rejected": -2432.0,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.625,
      "rewards/margins": 12.0,
      "rewards/rejected": -21.625,
      "step": 4150
    },
    {
      "epoch": 4.35374149659864,
      "grad_norm": 0.018877951979946545,
      "learning_rate": 2.484849537003184e-06,
      "logits/chosen": -0.7578125,
      "logits/rejected": 2.765625,
      "logps/chosen": -1288.0,
      "logps/rejected": -2464.0,
      "loss": 0.008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.75,
      "rewards/margins": 11.8125,
      "rewards/rejected": -21.625,
      "step": 4160
    },
    {
      "epoch": 4.364207221350078,
      "grad_norm": 0.012654241651534965,
      "learning_rate": 2.406009906248169e-06,
      "logits/chosen": -0.208984375,
      "logits/rejected": 3.640625,
      "logps/chosen": -1416.0,
      "logps/rejected": -2544.0,
      "loss": 0.0055,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -10.4375,
      "rewards/margins": 12.0625,
      "rewards/rejected": -22.5,
      "step": 4170
    },
    {
      "epoch": 4.3746729461015175,
      "grad_norm": 0.015622471752332513,
      "learning_rate": 2.328377978193924e-06,
      "logits/chosen": -0.020751953125,
      "logits/rejected": 3.328125,
      "logps/chosen": -1304.0,
      "logps/rejected": -2480.0,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.75,
      "rewards/margins": 12.0625,
      "rewards/rejected": -21.75,
      "step": 4180
    },
    {
      "epoch": 4.385138670852957,
      "grad_norm": 0.030115198260313527,
      "learning_rate": 2.2519579024523035e-06,
      "logits/chosen": 0.2890625,
      "logits/rejected": 4.21875,
      "logps/chosen": -1264.0,
      "logps/rejected": -2496.0,
      "loss": 0.008,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.6875,
      "rewards/margins": 12.5625,
      "rewards/rejected": -22.25,
      "step": 4190
    },
    {
      "epoch": 4.395604395604396,
      "grad_norm": 0.010975380556523135,
      "learning_rate": 2.1767537638587743e-06,
      "logits/chosen": 0.78515625,
      "logits/rejected": 4.15625,
      "logps/chosen": -1376.0,
      "logps/rejected": -2560.0,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -10.375,
      "rewards/margins": 12.1875,
      "rewards/rejected": -22.625,
      "step": 4200
    },
    {
      "epoch": 4.395604395604396,
      "eval_logits/chosen": 1.8125,
      "eval_logits/rejected": 3.75,
      "eval_logps/chosen": -1648.0,
      "eval_logps/rejected": -1952.0,
      "eval_loss": 1.1394531726837158,
      "eval_rewards/accuracies": 0.70703125,
      "eval_rewards/chosen": -13.3125,
      "eval_rewards/margins": 3.3125,
      "eval_rewards/rejected": -16.625,
      "eval_runtime": 52.8795,
      "eval_samples_per_second": 37.822,
      "eval_steps_per_second": 0.605,
      "step": 4200
    },
    {
      "epoch": 4.406070120355834,
      "grad_norm": 0.010336791984861466,
      "learning_rate": 2.1027695822540493e-06,
      "logits/chosen": 0.291015625,
      "logits/rejected": 4.03125,
      "logps/chosen": -1280.0,
      "logps/rejected": -2464.0,
      "loss": 0.0051,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.6875,
      "rewards/margins": 12.0625,
      "rewards/rejected": -21.75,
      "step": 4210
    },
    {
      "epoch": 4.416535845107274,
      "grad_norm": 0.022113022114078094,
      "learning_rate": 2.0300093122692184e-06,
      "logits/chosen": 1.0390625,
      "logits/rejected": 4.71875,
      "logps/chosen": -1320.0,
      "logps/rejected": -2496.0,
      "loss": 0.0047,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -10.0625,
      "rewards/margins": 12.125,
      "rewards/rejected": -22.25,
      "step": 4220
    },
    {
      "epoch": 4.427001569858713,
      "grad_norm": 0.046000333165739614,
      "learning_rate": 1.9584768431143955e-06,
      "logits/chosen": -0.1279296875,
      "logits/rejected": 3.765625,
      "logps/chosen": -1320.0,
      "logps/rejected": -2432.0,
      "loss": 0.0058,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.0,
      "rewards/margins": 11.8125,
      "rewards/rejected": -21.875,
      "step": 4230
    },
    {
      "epoch": 4.437467294610152,
      "grad_norm": 0.03154170065295428,
      "learning_rate": 1.888175998370803e-06,
      "logits/chosen": 0.43359375,
      "logits/rejected": 4.09375,
      "logps/chosen": -1264.0,
      "logps/rejected": -2368.0,
      "loss": 0.0092,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.6875,
      "rewards/margins": 11.375,
      "rewards/rejected": -21.0,
      "step": 4240
    },
    {
      "epoch": 4.4479330193615905,
      "grad_norm": 0.02632161963106536,
      "learning_rate": 1.819110535786389e-06,
      "logits/chosen": 0.1044921875,
      "logits/rejected": 3.890625,
      "logps/chosen": -1328.0,
      "logps/rejected": -2448.0,
      "loss": 0.0069,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.0625,
      "rewards/margins": 11.625,
      "rewards/rejected": -21.75,
      "step": 4250
    },
    {
      "epoch": 4.45839874411303,
      "grad_norm": 0.01806533303901749,
      "learning_rate": 1.751284147074983e-06,
      "logits/chosen": 0.7109375,
      "logits/rejected": 3.96875,
      "logps/chosen": -1376.0,
      "logps/rejected": -2544.0,
      "loss": 0.0075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.625,
      "rewards/margins": 11.8125,
      "rewards/rejected": -22.375,
      "step": 4260
    },
    {
      "epoch": 4.468864468864469,
      "grad_norm": 0.01073139614027261,
      "learning_rate": 1.6847004577189656e-06,
      "logits/chosen": 0.1904296875,
      "logits/rejected": 3.515625,
      "logps/chosen": -1408.0,
      "logps/rejected": -2576.0,
      "loss": 0.0039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.5625,
      "rewards/margins": 12.0,
      "rewards/rejected": -22.625,
      "step": 4270
    },
    {
      "epoch": 4.479330193615908,
      "grad_norm": 0.05597611228727731,
      "learning_rate": 1.6193630267754555e-06,
      "logits/chosen": -0.0179443359375,
      "logits/rejected": 3.359375,
      "logps/chosen": -1296.0,
      "logps/rejected": -2480.0,
      "loss": 0.0047,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.8125,
      "rewards/margins": 12.25,
      "rewards/rejected": -22.125,
      "step": 4280
    },
    {
      "epoch": 4.489795918367347,
      "grad_norm": 0.014723482512688289,
      "learning_rate": 1.5552753466860998e-06,
      "logits/chosen": 0.6328125,
      "logits/rejected": 4.40625,
      "logps/chosen": -1304.0,
      "logps/rejected": -2448.0,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.0,
      "rewards/margins": 11.8125,
      "rewards/rejected": -21.875,
      "step": 4290
    },
    {
      "epoch": 4.500261643118786,
      "grad_norm": 0.21716600651481588,
      "learning_rate": 1.4924408430903648e-06,
      "logits/chosen": -0.421875,
      "logits/rejected": 2.34375,
      "logps/chosen": -1360.0,
      "logps/rejected": -2544.0,
      "loss": 0.0042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.3125,
      "rewards/margins": 11.9375,
      "rewards/rejected": -22.25,
      "step": 4300
    },
    {
      "epoch": 4.500261643118786,
      "eval_logits/chosen": 1.8671875,
      "eval_logits/rejected": 3.796875,
      "eval_logps/chosen": -1664.0,
      "eval_logps/rejected": -1968.0,
      "eval_loss": 1.1511952877044678,
      "eval_rewards/accuracies": 0.705078125,
      "eval_rewards/chosen": -13.4375,
      "eval_rewards/margins": 3.34375,
      "eval_rewards/rejected": -16.75,
      "eval_runtime": 52.8769,
      "eval_samples_per_second": 37.824,
      "eval_steps_per_second": 0.605,
      "step": 4300
    },
    {
      "epoch": 4.510727367870225,
      "grad_norm": 0.026913537093199838,
      "learning_rate": 1.4308628746424496e-06,
      "logits/chosen": 0.263671875,
      "logits/rejected": 3.5625,
      "logps/chosen": -1296.0,
      "logps/rejected": -2464.0,
      "loss": 0.0081,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.125,
      "rewards/margins": 11.75,
      "rewards/rejected": -21.875,
      "step": 4310
    },
    {
      "epoch": 4.521193092621664,
      "grad_norm": 0.00939530700261345,
      "learning_rate": 1.3705447328317579e-06,
      "logits/chosen": 0.154296875,
      "logits/rejected": 3.953125,
      "logps/chosen": -1368.0,
      "logps/rejected": -2496.0,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -10.375,
      "rewards/margins": 11.75,
      "rewards/rejected": -22.125,
      "step": 4320
    },
    {
      "epoch": 4.531658817373103,
      "grad_norm": 0.0364511094649996,
      "learning_rate": 1.3114896418069372e-06,
      "logits/chosen": -0.23046875,
      "logits/rejected": 3.609375,
      "logps/chosen": -1360.0,
      "logps/rejected": -2432.0,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.8125,
      "rewards/margins": 11.5,
      "rewards/rejected": -21.25,
      "step": 4330
    },
    {
      "epoch": 4.542124542124542,
      "grad_norm": 0.02113594650850916,
      "learning_rate": 1.2537007582035631e-06,
      "logits/chosen": 0.6875,
      "logits/rejected": 4.40625,
      "logps/chosen": -1400.0,
      "logps/rejected": -2560.0,
      "loss": 0.0091,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -10.6875,
      "rewards/margins": 12.0625,
      "rewards/rejected": -22.75,
      "step": 4340
    },
    {
      "epoch": 4.552590266875981,
      "grad_norm": 0.05586788112085024,
      "learning_rate": 1.1971811709754089e-06,
      "logits/chosen": -0.056396484375,
      "logits/rejected": 4.0625,
      "logps/chosen": -1304.0,
      "logps/rejected": -2496.0,
      "loss": 0.0047,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.0625,
      "rewards/margins": 12.125,
      "rewards/rejected": -22.25,
      "step": 4350
    },
    {
      "epoch": 4.5630559916274205,
      "grad_norm": 0.05353120963273352,
      "learning_rate": 1.1419339012293112e-06,
      "logits/chosen": 0.1982421875,
      "logits/rejected": 4.03125,
      "logps/chosen": -1336.0,
      "logps/rejected": -2432.0,
      "loss": 0.0084,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.125,
      "rewards/margins": 11.4375,
      "rewards/rejected": -21.625,
      "step": 4360
    },
    {
      "epoch": 4.573521716378859,
      "grad_norm": 0.018632606756551547,
      "learning_rate": 1.0879619020637167e-06,
      "logits/chosen": 0.51953125,
      "logits/rejected": 4.59375,
      "logps/chosen": -1280.0,
      "logps/rejected": -2448.0,
      "loss": 0.007,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.8125,
      "rewards/margins": 11.875,
      "rewards/rejected": -21.625,
      "step": 4370
    },
    {
      "epoch": 4.583987441130298,
      "grad_norm": 0.35582959051096275,
      "learning_rate": 1.0352680584108042e-06,
      "logits/chosen": 0.3046875,
      "logits/rejected": 3.921875,
      "logps/chosen": -1304.0,
      "logps/rejected": -2448.0,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.0625,
      "rewards/margins": 11.8125,
      "rewards/rejected": -21.875,
      "step": 4380
    },
    {
      "epoch": 4.594453165881737,
      "grad_norm": 0.024948404025571833,
      "learning_rate": 9.838551868822931e-07,
      "logits/chosen": 0.255859375,
      "logits/rejected": 4.4375,
      "logps/chosen": -1344.0,
      "logps/rejected": -2528.0,
      "loss": 0.0048,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.125,
      "rewards/margins": 12.4375,
      "rewards/rejected": -22.625,
      "step": 4390
    },
    {
      "epoch": 4.604918890633177,
      "grad_norm": 0.015692333357294058,
      "learning_rate": 9.337260356188899e-07,
      "logits/chosen": 0.330078125,
      "logits/rejected": 4.40625,
      "logps/chosen": -1328.0,
      "logps/rejected": -2512.0,
      "loss": 0.0094,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -10.0625,
      "rewards/margins": 12.4375,
      "rewards/rejected": -22.5,
      "step": 4400
    },
    {
      "epoch": 4.604918890633177,
      "eval_logits/chosen": 1.8828125,
      "eval_logits/rejected": 3.8125,
      "eval_logps/chosen": -1664.0,
      "eval_logps/rejected": -1976.0,
      "eval_loss": 1.157953143119812,
      "eval_rewards/accuracies": 0.70703125,
      "eval_rewards/chosen": -13.5,
      "eval_rewards/margins": 3.359375,
      "eval_rewards/rejected": -16.875,
      "eval_runtime": 52.8911,
      "eval_samples_per_second": 37.814,
      "eval_steps_per_second": 0.605,
      "step": 4400
    },
    {
      "epoch": 4.615384615384615,
      "grad_norm": 0.014058491846490884,
      "learning_rate": 8.848832841433813e-07,
      "logits/chosen": 0.0361328125,
      "logits/rejected": 4.09375,
      "logps/chosen": -1328.0,
      "logps/rejected": -2464.0,
      "loss": 0.0061,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.9375,
      "rewards/margins": 12.0,
      "rewards/rejected": -21.875,
      "step": 4410
    },
    {
      "epoch": 4.625850340136054,
      "grad_norm": 0.02452867875576879,
      "learning_rate": 8.373295432174161e-07,
      "logits/chosen": 0.181640625,
      "logits/rejected": 3.640625,
      "logps/chosen": -1264.0,
      "logps/rejected": -2400.0,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.625,
      "rewards/margins": 11.4375,
      "rewards/rejected": -21.0,
      "step": 4420
    },
    {
      "epoch": 4.6363160648874935,
      "grad_norm": 0.014604552527745927,
      "learning_rate": 7.910673547019576e-07,
      "logits/chosen": 0.06689453125,
      "logits/rejected": 3.453125,
      "logps/chosen": -1352.0,
      "logps/rejected": -2480.0,
      "loss": 0.0076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.25,
      "rewards/margins": 11.5625,
      "rewards/rejected": -21.875,
      "step": 4430
    },
    {
      "epoch": 4.646781789638933,
      "grad_norm": 0.14235746342352668,
      "learning_rate": 7.460991914214066e-07,
      "logits/chosen": 0.4921875,
      "logits/rejected": 4.09375,
      "logps/chosen": -1360.0,
      "logps/rejected": -2544.0,
      "loss": 0.006,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.1875,
      "rewards/margins": 12.4375,
      "rewards/rejected": -22.625,
      "step": 4440
    },
    {
      "epoch": 4.657247514390372,
      "grad_norm": 0.18018234370057737,
      "learning_rate": 7.024274570314183e-07,
      "logits/chosen": 0.1396484375,
      "logits/rejected": 3.828125,
      "logps/chosen": -1280.0,
      "logps/rejected": -2528.0,
      "loss": 0.0106,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.75,
      "rewards/margins": 12.8125,
      "rewards/rejected": -22.625,
      "step": 4450
    },
    {
      "epoch": 4.66771323914181,
      "grad_norm": 0.022964972142676853,
      "learning_rate": 6.600544858904489e-07,
      "logits/chosen": 0.2392578125,
      "logits/rejected": 4.625,
      "logps/chosen": -1328.0,
      "logps/rejected": -2384.0,
      "loss": 0.0057,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.875,
      "rewards/margins": 11.1875,
      "rewards/rejected": -21.125,
      "step": 4460
    },
    {
      "epoch": 4.67817896389325,
      "grad_norm": 0.028318816060594693,
      "learning_rate": 6.189825429349399e-07,
      "logits/chosen": 0.04638671875,
      "logits/rejected": 3.75,
      "logps/chosen": -1400.0,
      "logps/rejected": -2496.0,
      "loss": 0.0058,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.375,
      "rewards/margins": 11.625,
      "rewards/rejected": -22.0,
      "step": 4470
    },
    {
      "epoch": 4.688644688644689,
      "grad_norm": 0.024191997780339095,
      "learning_rate": 5.792138235582834e-07,
      "logits/chosen": 0.1025390625,
      "logits/rejected": 3.328125,
      "logps/chosen": -1280.0,
      "logps/rejected": -2448.0,
      "loss": 0.0051,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.875,
      "rewards/margins": 11.9375,
      "rewards/rejected": -21.75,
      "step": 4480
    },
    {
      "epoch": 4.699110413396127,
      "grad_norm": 0.009190935628145188,
      "learning_rate": 5.407504534934493e-07,
      "logits/chosen": 0.443359375,
      "logits/rejected": 3.59375,
      "logps/chosen": -1352.0,
      "logps/rejected": -2576.0,
      "loss": 0.0051,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.125,
      "rewards/margins": 12.375,
      "rewards/rejected": -22.5,
      "step": 4490
    },
    {
      "epoch": 4.7095761381475665,
      "grad_norm": 0.016416213341888905,
      "learning_rate": 5.03594488699377e-07,
      "logits/chosen": 0.333984375,
      "logits/rejected": 3.828125,
      "logps/chosen": -1376.0,
      "logps/rejected": -2592.0,
      "loss": 0.006,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -10.4375,
      "rewards/margins": 12.375,
      "rewards/rejected": -22.875,
      "step": 4500
    },
    {
      "epoch": 4.7095761381475665,
      "eval_logits/chosen": 1.921875,
      "eval_logits/rejected": 3.84375,
      "eval_logps/chosen": -1672.0,
      "eval_logps/rejected": -1984.0,
      "eval_loss": 1.1593124866485596,
      "eval_rewards/accuracies": 0.705078125,
      "eval_rewards/chosen": -13.5625,
      "eval_rewards/margins": 3.375,
      "eval_rewards/rejected": -17.0,
      "eval_runtime": 52.8695,
      "eval_samples_per_second": 37.829,
      "eval_steps_per_second": 0.605,
      "step": 4500
    },
    {
      "epoch": 4.720041862899006,
      "grad_norm": 0.01899683797190987,
      "learning_rate": 4.677479152510794e-07,
      "logits/chosen": 0.45703125,
      "logits/rejected": 3.828125,
      "logps/chosen": -1280.0,
      "logps/rejected": -2592.0,
      "loss": 0.0037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.875,
      "rewards/margins": 13.1875,
      "rewards/rejected": -23.0,
      "step": 4510
    },
    {
      "epoch": 4.730507587650445,
      "grad_norm": 0.01647864082043874,
      "learning_rate": 4.3321264923346925e-07,
      "logits/chosen": -0.22265625,
      "logits/rejected": 3.90625,
      "logps/chosen": -1320.0,
      "logps/rejected": -2544.0,
      "loss": 0.0026,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.0625,
      "rewards/margins": 12.5625,
      "rewards/rejected": -22.625,
      "step": 4520
    },
    {
      "epoch": 4.740973312401884,
      "grad_norm": 0.013445849354382783,
      "learning_rate": 3.9999053663893893e-07,
      "logits/chosen": 0.0693359375,
      "logits/rejected": 4.3125,
      "logps/chosen": -1384.0,
      "logps/rejected": -2608.0,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.5,
      "rewards/margins": 12.75,
      "rewards/rejected": -23.25,
      "step": 4530
    },
    {
      "epoch": 4.751439037153323,
      "grad_norm": 0.022597491720675638,
      "learning_rate": 3.6808335326871944e-07,
      "logits/chosen": 0.1591796875,
      "logits/rejected": 4.625,
      "logps/chosen": -1360.0,
      "logps/rejected": -2512.0,
      "loss": 0.0058,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -10.375,
      "rewards/margins": 12.0,
      "rewards/rejected": -22.375,
      "step": 4540
    },
    {
      "epoch": 4.761904761904762,
      "grad_norm": 0.02691200162294267,
      "learning_rate": 3.374928046379122e-07,
      "logits/chosen": 0.625,
      "logits/rejected": 4.78125,
      "logps/chosen": -1320.0,
      "logps/rejected": -2400.0,
      "loss": 0.0072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.0,
      "rewards/margins": 11.5625,
      "rewards/rejected": -21.625,
      "step": 4550
    },
    {
      "epoch": 4.772370486656201,
      "grad_norm": 0.06724491422656781,
      "learning_rate": 3.082205258843618e-07,
      "logits/chosen": -0.020263671875,
      "logits/rejected": 3.859375,
      "logps/chosen": -1336.0,
      "logps/rejected": -2432.0,
      "loss": 0.009,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -10.25,
      "rewards/margins": 11.625,
      "rewards/rejected": -21.875,
      "step": 4560
    },
    {
      "epoch": 4.7828362114076395,
      "grad_norm": 0.013820088346344305,
      "learning_rate": 2.8026808168123973e-07,
      "logits/chosen": 0.318359375,
      "logits/rejected": 4.09375,
      "logps/chosen": -1304.0,
      "logps/rejected": -2480.0,
      "loss": 0.0115,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.875,
      "rewards/margins": 12.0625,
      "rewards/rejected": -21.875,
      "step": 4570
    },
    {
      "epoch": 4.793301936159079,
      "grad_norm": 0.015977450255060588,
      "learning_rate": 2.5363696615340883e-07,
      "logits/chosen": -0.28125,
      "logits/rejected": 3.765625,
      "logps/chosen": -1384.0,
      "logps/rejected": -2512.0,
      "loss": 0.0076,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -10.4375,
      "rewards/margins": 11.625,
      "rewards/rejected": -22.125,
      "step": 4580
    },
    {
      "epoch": 4.803767660910518,
      "grad_norm": 0.19866355825297102,
      "learning_rate": 2.2832860279756174e-07,
      "logits/chosen": 0.4921875,
      "logits/rejected": 4.34375,
      "logps/chosen": -1352.0,
      "logps/rejected": -2528.0,
      "loss": 0.0055,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.3125,
      "rewards/margins": 12.0625,
      "rewards/rejected": -22.375,
      "step": 4590
    },
    {
      "epoch": 4.814233385661957,
      "grad_norm": 0.14021973363064155,
      "learning_rate": 2.0434434440612381e-07,
      "logits/chosen": 0.6953125,
      "logits/rejected": 4.53125,
      "logps/chosen": -1280.0,
      "logps/rejected": -2464.0,
      "loss": 0.0029,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -10.0625,
      "rewards/margins": 12.0,
      "rewards/rejected": -22.125,
      "step": 4600
    },
    {
      "epoch": 4.814233385661957,
      "eval_logits/chosen": 1.9375,
      "eval_logits/rejected": 3.859375,
      "eval_logps/chosen": -1680.0,
      "eval_logps/rejected": -1984.0,
      "eval_loss": 1.1617499589920044,
      "eval_rewards/accuracies": 0.705078125,
      "eval_rewards/chosen": -13.625,
      "eval_rewards/margins": 3.375,
      "eval_rewards/rejected": -17.0,
      "eval_runtime": 52.8851,
      "eval_samples_per_second": 37.818,
      "eval_steps_per_second": 0.605,
      "step": 4600
    },
    {
      "epoch": 4.8246991104133965,
      "grad_norm": 0.013472860117799914,
      "learning_rate": 1.8168547299496076e-07,
      "logits/chosen": -0.25,
      "logits/rejected": 3.8125,
      "logps/chosen": -1344.0,
      "logps/rejected": -2448.0,
      "loss": 0.0029,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.0,
      "rewards/margins": 11.75,
      "rewards/rejected": -21.75,
      "step": 4610
    },
    {
      "epoch": 4.835164835164835,
      "grad_norm": 0.022340399767170387,
      "learning_rate": 1.6035319973483342e-07,
      "logits/chosen": -0.67578125,
      "logits/rejected": 3.90625,
      "logps/chosen": -1352.0,
      "logps/rejected": -2544.0,
      "loss": 0.0037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.25,
      "rewards/margins": 12.4375,
      "rewards/rejected": -22.625,
      "step": 4620
    },
    {
      "epoch": 4.845630559916274,
      "grad_norm": 0.05257551803550983,
      "learning_rate": 1.4034866488666364e-07,
      "logits/chosen": -0.255859375,
      "logits/rejected": 3.453125,
      "logps/chosen": -1352.0,
      "logps/rejected": -2560.0,
      "loss": 0.0037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.0,
      "rewards/margins": 12.6875,
      "rewards/rejected": -22.75,
      "step": 4630
    },
    {
      "epoch": 4.856096284667713,
      "grad_norm": 0.02207698637345573,
      "learning_rate": 1.2167293774059118e-07,
      "logits/chosen": 1.2421875,
      "logits/rejected": 4.65625,
      "logps/chosen": -1328.0,
      "logps/rejected": -2544.0,
      "loss": 0.0041,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.3125,
      "rewards/margins": 12.4375,
      "rewards/rejected": -22.75,
      "step": 4640
    },
    {
      "epoch": 4.866562009419153,
      "grad_norm": 0.02412749963928652,
      "learning_rate": 1.0432701655880838e-07,
      "logits/chosen": 0.5859375,
      "logits/rejected": 3.5625,
      "logps/chosen": -1352.0,
      "logps/rejected": -2560.0,
      "loss": 0.009,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -10.0625,
      "rewards/margins": 12.1875,
      "rewards/rejected": -22.25,
      "step": 4650
    },
    {
      "epoch": 4.877027734170591,
      "grad_norm": 0.025434985246309914,
      "learning_rate": 8.831182852220565e-08,
      "logits/chosen": -0.8046875,
      "logits/rejected": 3.25,
      "logps/chosen": -1384.0,
      "logps/rejected": -2544.0,
      "loss": 0.0036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.25,
      "rewards/margins": 12.25,
      "rewards/rejected": -22.5,
      "step": 4660
    },
    {
      "epoch": 4.88749345892203,
      "grad_norm": 0.016137357714564624,
      "learning_rate": 7.362822968081107e-08,
      "logits/chosen": 0.7109375,
      "logits/rejected": 4.15625,
      "logps/chosen": -1264.0,
      "logps/rejected": -2432.0,
      "loss": 0.0075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.75,
      "rewards/margins": 12.0,
      "rewards/rejected": -21.75,
      "step": 4670
    },
    {
      "epoch": 4.8979591836734695,
      "grad_norm": 0.015816330428665797,
      "learning_rate": 6.02770049080298e-08,
      "logits/chosen": 0.5859375,
      "logits/rejected": 4.125,
      "logps/chosen": -1344.0,
      "logps/rejected": -2528.0,
      "loss": 0.0081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -10.1875,
      "rewards/margins": 12.125,
      "rewards/rejected": -22.25,
      "step": 4680
    },
    {
      "epoch": 4.908424908424909,
      "grad_norm": 0.01587861482983314,
      "learning_rate": 4.8258867858697044e-08,
      "logits/chosen": 0.58203125,
      "logits/rejected": 3.21875,
      "logps/chosen": -1272.0,
      "logps/rejected": -2528.0,
      "loss": 0.0027,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.0625,
      "rewards/margins": 12.4375,
      "rewards/rejected": -22.5,
      "step": 4690
    },
    {
      "epoch": 4.918890633176347,
      "grad_norm": 0.01701742299215144,
      "learning_rate": 3.757446093092254e-08,
      "logits/chosen": 0.93359375,
      "logits/rejected": 4.4375,
      "logps/chosen": -1432.0,
      "logps/rejected": -2576.0,
      "loss": 0.0059,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.9375,
      "rewards/margins": 11.875,
      "rewards/rejected": -22.875,
      "step": 4700
    },
    {
      "epoch": 4.918890633176347,
      "eval_logits/chosen": 1.9453125,
      "eval_logits/rejected": 3.859375,
      "eval_logps/chosen": -1680.0,
      "eval_logps/rejected": -1984.0,
      "eval_loss": 1.1636874675750732,
      "eval_rewards/accuracies": 0.705078125,
      "eval_rewards/chosen": -13.625,
      "eval_rewards/margins": 3.375,
      "eval_rewards/rejected": -17.0,
      "eval_runtime": 52.896,
      "eval_samples_per_second": 37.81,
      "eval_steps_per_second": 0.605,
      "step": 4700
    },
    {
      "epoch": 4.929356357927786,
      "grad_norm": 0.009136301419989667,
      "learning_rate": 2.8224355231762368e-08,
      "logits/chosen": 0.54296875,
      "logits/rejected": 4.34375,
      "logps/chosen": -1320.0,
      "logps/rejected": -2544.0,
      "loss": 0.006,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.0625,
      "rewards/margins": 12.5,
      "rewards/rejected": -22.625,
      "step": 4710
    },
    {
      "epoch": 4.939822082679226,
      "grad_norm": 0.016109187019904162,
      "learning_rate": 2.020905054668232e-08,
      "logits/chosen": 0.478515625,
      "logits/rejected": 3.921875,
      "logps/chosen": -1360.0,
      "logps/rejected": -2544.0,
      "loss": 0.0079,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -10.125,
      "rewards/margins": 12.1875,
      "rewards/rejected": -22.375,
      "step": 4720
    },
    {
      "epoch": 4.950287807430665,
      "grad_norm": 0.08635472918390723,
      "learning_rate": 1.3528975312848691e-08,
      "logits/chosen": 0.72265625,
      "logits/rejected": 3.78125,
      "logps/chosen": -1360.0,
      "logps/rejected": -2464.0,
      "loss": 0.0028,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.3125,
      "rewards/margins": 11.375,
      "rewards/rejected": -21.625,
      "step": 4730
    },
    {
      "epoch": 4.960753532182103,
      "grad_norm": 0.13889460429320133,
      "learning_rate": 8.18448659622717e-09,
      "logits/chosen": -0.216796875,
      "logits/rejected": 3.5,
      "logps/chosen": -1360.0,
      "logps/rejected": -2528.0,
      "loss": 0.0038,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.5,
      "rewards/margins": 11.9375,
      "rewards/rejected": -22.375,
      "step": 4740
    },
    {
      "epoch": 4.9712192569335425,
      "grad_norm": 0.054086673578341196,
      "learning_rate": 4.175870072500865e-09,
      "logits/chosen": -0.1767578125,
      "logits/rejected": 3.125,
      "logps/chosen": -1336.0,
      "logps/rejected": -2400.0,
      "loss": 0.0048,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.0,
      "rewards/margins": 11.1875,
      "rewards/rejected": -21.25,
      "step": 4750
    },
    {
      "epoch": 4.981684981684982,
      "grad_norm": 0.021898955068035898,
      "learning_rate": 1.5033400117853146e-09,
      "logits/chosen": 0.3046875,
      "logits/rejected": 3.90625,
      "logps/chosen": -1296.0,
      "logps/rejected": -2416.0,
      "loss": 0.0035,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -9.875,
      "rewards/margins": 11.5625,
      "rewards/rejected": -21.5,
      "step": 4760
    },
    {
      "epoch": 4.992150706436421,
      "grad_norm": 0.02157469228444244,
      "learning_rate": 1.670392671959675e-10,
      "logits/chosen": 0.64453125,
      "logits/rejected": 4.34375,
      "logps/chosen": -1336.0,
      "logps/rejected": -2512.0,
      "loss": 0.0064,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.1875,
      "rewards/margins": 12.125,
      "rewards/rejected": -22.25,
      "step": 4770
    },
    {
      "epoch": 4.99738356881214,
      "step": 4775,
      "total_flos": 0.0,
      "train_loss": 0.1729966444779597,
      "train_runtime": 22065.583,
      "train_samples_per_second": 13.853,
      "train_steps_per_second": 0.216
    }
  ],
  "logging_steps": 10,
  "max_steps": 4775,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 5,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}