{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 936,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.003205128205128205,
      "grad_norm": 38.020048422719306,
      "learning_rate": 5.3191489361702125e-09,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -172.0,
      "logps/rejected": -160.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03205128205128205,
      "grad_norm": 34.89707580881474,
      "learning_rate": 5.3191489361702123e-08,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.4375,
      "logps/chosen": -140.0,
      "logps/rejected": -181.0,
      "loss": 0.691,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.00592041015625,
      "rewards/margins": -0.003997802734375,
      "rewards/rejected": -0.0019073486328125,
      "step": 10
    },
    {
      "epoch": 0.0641025641025641,
      "grad_norm": 35.22039582592668,
      "learning_rate": 1.0638297872340425e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.40625,
      "logps/chosen": -116.0,
      "logps/rejected": -171.0,
      "loss": 0.6797,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.0260009765625,
      "rewards/margins": 0.0264892578125,
      "rewards/rejected": -0.0004749298095703125,
      "step": 20
    },
    {
      "epoch": 0.09615384615384616,
      "grad_norm": 40.58215068588761,
      "learning_rate": 1.5957446808510638e-07,
      "logits/chosen": -1.46875,
      "logits/rejected": -1.4140625,
      "logps/chosen": -122.5,
      "logps/rejected": -180.0,
      "loss": 0.6562,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.11474609375,
      "rewards/margins": 0.08203125,
      "rewards/rejected": 0.03271484375,
      "step": 30
    },
    {
      "epoch": 0.1282051282051282,
      "grad_norm": 35.55857804315714,
      "learning_rate": 2.127659574468085e-07,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.4609375,
      "logps/chosen": -127.5,
      "logps/rejected": -180.0,
      "loss": 0.6092,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.259765625,
      "rewards/margins": 0.21484375,
      "rewards/rejected": 0.046142578125,
      "step": 40
    },
    {
      "epoch": 0.16025641025641027,
      "grad_norm": 23.579694487139133,
      "learning_rate": 2.659574468085106e-07,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.453125,
      "logps/chosen": -116.5,
      "logps/rejected": -174.0,
      "loss": 0.5579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.26953125,
      "rewards/margins": 0.365234375,
      "rewards/rejected": -0.095703125,
      "step": 50
    },
    {
      "epoch": 0.19230769230769232,
      "grad_norm": 23.19132387624345,
      "learning_rate": 3.1914893617021275e-07,
      "logits/chosen": -1.453125,
      "logits/rejected": -1.40625,
      "logps/chosen": -99.5,
      "logps/rejected": -155.0,
      "loss": 0.5053,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.298828125,
      "rewards/margins": 0.6015625,
      "rewards/rejected": -0.30078125,
      "step": 60
    },
    {
      "epoch": 0.22435897435897437,
      "grad_norm": 21.818865607569602,
      "learning_rate": 3.7234042553191484e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.484375,
      "logps/chosen": -100.5,
      "logps/rejected": -162.0,
      "loss": 0.3894,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.3984375,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -0.66796875,
      "step": 70
    },
    {
      "epoch": 0.2564102564102564,
      "grad_norm": 26.315987904129013,
      "learning_rate": 4.25531914893617e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.4921875,
      "logps/chosen": -117.5,
      "logps/rejected": -170.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.026611328125,
      "rewards/margins": 1.1875,
      "rewards/rejected": -1.1640625,
      "step": 80
    },
    {
      "epoch": 0.28846153846153844,
      "grad_norm": 28.51531224072289,
      "learning_rate": 4.787234042553192e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.515625,
      "logps/chosen": -114.5,
      "logps/rejected": -176.0,
      "loss": 0.387,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1240234375,
      "rewards/margins": 1.375,
      "rewards/rejected": -1.5,
      "step": 90
    },
    {
      "epoch": 0.32051282051282054,
      "grad_norm": 17.39940529070616,
      "learning_rate": 4.96437054631829e-07,
      "logits/chosen": -1.59375,
      "logits/rejected": -1.5234375,
      "logps/chosen": -105.5,
      "logps/rejected": -169.0,
      "loss": 0.324,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.390625,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -1.2734375,
      "step": 100
    },
    {
      "epoch": 0.3525641025641026,
      "grad_norm": 28.443023292937838,
      "learning_rate": 4.904988123515439e-07,
      "logits/chosen": -1.6015625,
      "logits/rejected": -1.5546875,
      "logps/chosen": -118.0,
      "logps/rejected": -199.0,
      "loss": 0.3077,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.052490234375,
      "rewards/margins": 1.75,
      "rewards/rejected": -1.703125,
      "step": 110
    },
    {
      "epoch": 0.38461538461538464,
      "grad_norm": 19.86248937078093,
      "learning_rate": 4.845605700712589e-07,
      "logits/chosen": -1.640625,
      "logits/rejected": -1.515625,
      "logps/chosen": -118.0,
      "logps/rejected": -182.0,
      "loss": 0.3212,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.41015625,
      "rewards/margins": 2.0625,
      "rewards/rejected": -2.46875,
      "step": 120
    },
    {
      "epoch": 0.4166666666666667,
      "grad_norm": 15.34168051684897,
      "learning_rate": 4.786223277909738e-07,
      "logits/chosen": -1.625,
      "logits/rejected": -1.515625,
      "logps/chosen": -127.0,
      "logps/rejected": -196.0,
      "loss": 0.3436,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.73046875,
      "rewards/margins": 2.25,
      "rewards/rejected": -2.984375,
      "step": 130
    },
    {
      "epoch": 0.44871794871794873,
      "grad_norm": 31.34040320177804,
      "learning_rate": 4.7268408551068883e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.515625,
      "logps/chosen": -138.0,
      "logps/rejected": -193.0,
      "loss": 0.335,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.453125,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -2.21875,
      "step": 140
    },
    {
      "epoch": 0.4807692307692308,
      "grad_norm": 39.09253912133659,
      "learning_rate": 4.667458432304038e-07,
      "logits/chosen": -1.640625,
      "logits/rejected": -1.6328125,
      "logps/chosen": -128.0,
      "logps/rejected": -249.0,
      "loss": 0.2853,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08056640625,
      "rewards/margins": 2.53125,
      "rewards/rejected": -2.609375,
      "step": 150
    },
    {
      "epoch": 0.5128205128205128,
      "grad_norm": 17.194231471019577,
      "learning_rate": 4.6080760095011875e-07,
      "logits/chosen": -1.6015625,
      "logits/rejected": -1.5859375,
      "logps/chosen": -107.0,
      "logps/rejected": -191.0,
      "loss": 0.2666,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.310546875,
      "rewards/margins": 2.71875,
      "rewards/rejected": -3.015625,
      "step": 160
    },
    {
      "epoch": 0.5448717948717948,
      "grad_norm": 18.722209877304238,
      "learning_rate": 4.548693586698337e-07,
      "logits/chosen": -1.578125,
      "logits/rejected": -1.59375,
      "logps/chosen": -115.0,
      "logps/rejected": -175.0,
      "loss": 0.2696,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.150390625,
      "rewards/margins": 2.234375,
      "rewards/rejected": -2.390625,
      "step": 170
    },
    {
      "epoch": 0.5769230769230769,
      "grad_norm": 21.42273705026191,
      "learning_rate": 4.4893111638954866e-07,
      "logits/chosen": -1.5859375,
      "logits/rejected": -1.6171875,
      "logps/chosen": -121.5,
      "logps/rejected": -203.0,
      "loss": 0.2899,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.248046875,
      "rewards/margins": 2.390625,
      "rewards/rejected": -2.640625,
      "step": 180
    },
    {
      "epoch": 0.6089743589743589,
      "grad_norm": 35.105792045016955,
      "learning_rate": 4.429928741092636e-07,
      "logits/chosen": -1.625,
      "logits/rejected": -1.546875,
      "logps/chosen": -126.5,
      "logps/rejected": -199.0,
      "loss": 0.2834,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.38671875,
      "rewards/margins": 2.609375,
      "rewards/rejected": -3.0,
      "step": 190
    },
    {
      "epoch": 0.6410256410256411,
      "grad_norm": 25.037051509827627,
      "learning_rate": 4.3705463182897863e-07,
      "logits/chosen": -1.59375,
      "logits/rejected": -1.5078125,
      "logps/chosen": -116.5,
      "logps/rejected": -171.0,
      "loss": 0.3022,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.59375,
      "rewards/margins": 2.15625,
      "rewards/rejected": -2.75,
      "step": 200
    },
    {
      "epoch": 0.6730769230769231,
      "grad_norm": 25.3426239738337,
      "learning_rate": 4.311163895486936e-07,
      "logits/chosen": -1.5859375,
      "logits/rejected": -1.5546875,
      "logps/chosen": -118.5,
      "logps/rejected": -185.0,
      "loss": 0.3209,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.21875,
      "rewards/margins": 2.1875,
      "rewards/rejected": -3.40625,
      "step": 210
    },
    {
      "epoch": 0.7051282051282052,
      "grad_norm": 41.23383631897795,
      "learning_rate": 4.251781472684085e-07,
      "logits/chosen": -1.6484375,
      "logits/rejected": -1.5546875,
      "logps/chosen": -148.0,
      "logps/rejected": -200.0,
      "loss": 0.2921,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.609375,
      "rewards/margins": 2.3125,
      "rewards/rejected": -2.921875,
      "step": 220
    },
    {
      "epoch": 0.7371794871794872,
      "grad_norm": 22.289875990424193,
      "learning_rate": 4.192399049881235e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.5859375,
      "logps/chosen": -127.0,
      "logps/rejected": -199.0,
      "loss": 0.2805,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.318359375,
      "rewards/margins": 2.578125,
      "rewards/rejected": -2.90625,
      "step": 230
    },
    {
      "epoch": 0.7692307692307693,
      "grad_norm": 29.25683917677673,
      "learning_rate": 4.1330166270783846e-07,
      "logits/chosen": -1.59375,
      "logits/rejected": -1.5390625,
      "logps/chosen": -108.5,
      "logps/rejected": -180.0,
      "loss": 0.2541,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.267578125,
      "rewards/margins": 2.765625,
      "rewards/rejected": -3.03125,
      "step": 240
    },
    {
      "epoch": 0.8012820512820513,
      "grad_norm": 28.339641539576366,
      "learning_rate": 4.0736342042755347e-07,
      "logits/chosen": -1.6484375,
      "logits/rejected": -1.5625,
      "logps/chosen": -120.0,
      "logps/rejected": -215.0,
      "loss": 0.257,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.50390625,
      "rewards/margins": 3.140625,
      "rewards/rejected": -3.640625,
      "step": 250
    },
    {
      "epoch": 0.8333333333333334,
      "grad_norm": 28.458003688730955,
      "learning_rate": 4.0142517814726837e-07,
      "logits/chosen": -1.6328125,
      "logits/rejected": -1.5703125,
      "logps/chosen": -124.0,
      "logps/rejected": -195.0,
      "loss": 0.2514,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.3203125,
      "rewards/margins": 2.6875,
      "rewards/rejected": -3.015625,
      "step": 260
    },
    {
      "epoch": 0.8653846153846154,
      "grad_norm": 19.712649681094366,
      "learning_rate": 3.9548693586698333e-07,
      "logits/chosen": -1.640625,
      "logits/rejected": -1.546875,
      "logps/chosen": -121.0,
      "logps/rejected": -177.0,
      "loss": 0.2635,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.11083984375,
      "rewards/margins": 2.453125,
      "rewards/rejected": -2.5625,
      "step": 270
    },
    {
      "epoch": 0.8974358974358975,
      "grad_norm": 46.09415755147797,
      "learning_rate": 3.8954869358669834e-07,
      "logits/chosen": -1.6640625,
      "logits/rejected": -1.546875,
      "logps/chosen": -118.5,
      "logps/rejected": -197.0,
      "loss": 0.2671,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0693359375,
      "rewards/margins": 2.78125,
      "rewards/rejected": -2.703125,
      "step": 280
    },
    {
      "epoch": 0.9294871794871795,
      "grad_norm": 25.186336345995475,
      "learning_rate": 3.836104513064133e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.5390625,
      "logps/chosen": -129.0,
      "logps/rejected": -203.0,
      "loss": 0.2957,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.03125,
      "rewards/margins": 2.21875,
      "rewards/rejected": -3.25,
      "step": 290
    },
    {
      "epoch": 0.9615384615384616,
      "grad_norm": 25.061643380626624,
      "learning_rate": 3.7767220902612825e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.578125,
      "logps/chosen": -134.0,
      "logps/rejected": -219.0,
      "loss": 0.2345,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.90234375,
      "rewards/margins": 2.671875,
      "rewards/rejected": -3.578125,
      "step": 300
    },
    {
      "epoch": 0.9935897435897436,
      "grad_norm": 44.17174735441787,
      "learning_rate": 3.717339667458432e-07,
      "logits/chosen": -1.640625,
      "logits/rejected": -1.5703125,
      "logps/chosen": -119.0,
      "logps/rejected": -192.0,
      "loss": 0.2734,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.2392578125,
      "rewards/margins": 2.734375,
      "rewards/rejected": -2.96875,
      "step": 310
    },
    {
      "epoch": 1.0,
      "eval_logits/chosen": -1.5546875,
      "eval_logits/rejected": -1.5234375,
      "eval_logps/chosen": -170.0,
      "eval_logps/rejected": -184.0,
      "eval_loss": 0.6114453077316284,
      "eval_rewards/accuracies": 0.5892857313156128,
      "eval_rewards/chosen": -1.53125,
      "eval_rewards/margins": 0.8984375,
      "eval_rewards/rejected": -2.4375,
      "eval_runtime": 14.9255,
      "eval_samples_per_second": 13.4,
      "eval_steps_per_second": 0.469,
      "step": 312
    },
    {
      "epoch": 1.0256410256410255,
      "grad_norm": 16.449029965031666,
      "learning_rate": 3.6579572446555817e-07,
      "logits/chosen": -1.59375,
      "logits/rejected": -1.578125,
      "logps/chosen": -117.0,
      "logps/rejected": -195.0,
      "loss": 0.1531,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.34765625,
      "rewards/margins": 3.1875,
      "rewards/rejected": -3.546875,
      "step": 320
    },
    {
      "epoch": 1.0576923076923077,
      "grad_norm": 6.562166239087373,
      "learning_rate": 3.598574821852731e-07,
      "logits/chosen": -1.59375,
      "logits/rejected": -1.5703125,
      "logps/chosen": -111.0,
      "logps/rejected": -187.0,
      "loss": 0.1452,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1923828125,
      "rewards/margins": 3.40625,
      "rewards/rejected": -3.609375,
      "step": 330
    },
    {
      "epoch": 1.0897435897435896,
      "grad_norm": 25.028970432454248,
      "learning_rate": 3.5391923990498813e-07,
      "logits/chosen": -1.6484375,
      "logits/rejected": -1.609375,
      "logps/chosen": -134.0,
      "logps/rejected": -219.0,
      "loss": 0.1345,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.69921875,
      "rewards/margins": 3.609375,
      "rewards/rejected": -4.3125,
      "step": 340
    },
    {
      "epoch": 1.1217948717948718,
      "grad_norm": 15.596444813277682,
      "learning_rate": 3.479809976247031e-07,
      "logits/chosen": -1.65625,
      "logits/rejected": -1.6015625,
      "logps/chosen": -130.0,
      "logps/rejected": -207.0,
      "loss": 0.1092,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.00677490234375,
      "rewards/margins": 3.828125,
      "rewards/rejected": -3.828125,
      "step": 350
    },
    {
      "epoch": 1.1538461538461537,
      "grad_norm": 22.37742199079234,
      "learning_rate": 3.42042755344418e-07,
      "logits/chosen": -1.5859375,
      "logits/rejected": -1.5703125,
      "logps/chosen": -136.0,
      "logps/rejected": -242.0,
      "loss": 0.0918,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.640625,
      "rewards/margins": 4.4375,
      "rewards/rejected": -5.09375,
      "step": 360
    },
    {
      "epoch": 1.185897435897436,
      "grad_norm": 27.25128799333949,
      "learning_rate": 3.36104513064133e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.59375,
      "logps/chosen": -154.0,
      "logps/rejected": -221.0,
      "loss": 0.1215,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.333984375,
      "rewards/margins": 4.46875,
      "rewards/rejected": -4.78125,
      "step": 370
    },
    {
      "epoch": 1.217948717948718,
      "grad_norm": 16.802650370434502,
      "learning_rate": 3.3016627078384796e-07,
      "logits/chosen": -1.625,
      "logits/rejected": -1.515625,
      "logps/chosen": -121.0,
      "logps/rejected": -184.0,
      "loss": 0.1051,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.75390625,
      "rewards/margins": 3.75,
      "rewards/rejected": -4.5,
      "step": 380
    },
    {
      "epoch": 1.25,
      "grad_norm": 18.14599606798184,
      "learning_rate": 3.2422802850356297e-07,
      "logits/chosen": -1.65625,
      "logits/rejected": -1.5625,
      "logps/chosen": -118.0,
      "logps/rejected": -203.0,
      "loss": 0.1691,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.4609375,
      "rewards/margins": 4.4375,
      "rewards/rejected": -4.875,
      "step": 390
    },
    {
      "epoch": 1.282051282051282,
      "grad_norm": 14.341477798166297,
      "learning_rate": 3.182897862232779e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.5546875,
      "logps/chosen": -112.0,
      "logps/rejected": -202.0,
      "loss": 0.1084,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.2578125,
      "rewards/margins": 4.65625,
      "rewards/rejected": -4.90625,
      "step": 400
    },
    {
      "epoch": 1.314102564102564,
      "grad_norm": 22.538352683910524,
      "learning_rate": 3.1235154394299283e-07,
      "logits/chosen": -1.671875,
      "logits/rejected": -1.625,
      "logps/chosen": -168.0,
      "logps/rejected": -270.0,
      "loss": 0.136,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.1796875,
      "rewards/margins": 4.71875,
      "rewards/rejected": -5.90625,
      "step": 410
    },
    {
      "epoch": 1.3461538461538463,
      "grad_norm": 7.69063972210861,
      "learning_rate": 3.0641330166270784e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.59375,
      "logps/chosen": -128.0,
      "logps/rejected": -232.0,
      "loss": 0.1123,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.0859375,
      "rewards/margins": 4.59375,
      "rewards/rejected": -5.6875,
      "step": 420
    },
    {
      "epoch": 1.3782051282051282,
      "grad_norm": 22.81215314331503,
      "learning_rate": 3.004750593824228e-07,
      "logits/chosen": -1.6328125,
      "logits/rejected": -1.5390625,
      "logps/chosen": -122.5,
      "logps/rejected": -209.0,
      "loss": 0.0971,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.33984375,
      "rewards/margins": 4.65625,
      "rewards/rejected": -5.0,
      "step": 430
    },
    {
      "epoch": 1.4102564102564101,
      "grad_norm": 16.88060293661355,
      "learning_rate": 2.9453681710213776e-07,
      "logits/chosen": -1.5859375,
      "logits/rejected": -1.4765625,
      "logps/chosen": -123.0,
      "logps/rejected": -205.0,
      "loss": 0.11,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4296875,
      "rewards/margins": 4.5625,
      "rewards/rejected": -5.0,
      "step": 440
    },
    {
      "epoch": 1.4423076923076923,
      "grad_norm": 14.494261083829612,
      "learning_rate": 2.885985748218527e-07,
      "logits/chosen": -1.546875,
      "logits/rejected": -1.5546875,
      "logps/chosen": -118.5,
      "logps/rejected": -221.0,
      "loss": 0.0896,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.0859375,
      "rewards/margins": 4.59375,
      "rewards/rejected": -5.6875,
      "step": 450
    },
    {
      "epoch": 1.4743589743589745,
      "grad_norm": 16.655718879118293,
      "learning_rate": 2.8266033254156767e-07,
      "logits/chosen": -1.59375,
      "logits/rejected": -1.5859375,
      "logps/chosen": -142.0,
      "logps/rejected": -256.0,
      "loss": 0.1071,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.25,
      "rewards/margins": 4.84375,
      "rewards/rejected": -6.09375,
      "step": 460
    },
    {
      "epoch": 1.5064102564102564,
      "grad_norm": 20.51874312804957,
      "learning_rate": 2.7672209026128263e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.53125,
      "logps/chosen": -132.0,
      "logps/rejected": -231.0,
      "loss": 0.1012,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.015625,
      "rewards/margins": 4.65625,
      "rewards/rejected": -5.6875,
      "step": 470
    },
    {
      "epoch": 1.5384615384615383,
      "grad_norm": 28.03019085733964,
      "learning_rate": 2.7078384798099764e-07,
      "logits/chosen": -1.4453125,
      "logits/rejected": -1.4609375,
      "logps/chosen": -104.5,
      "logps/rejected": -203.0,
      "loss": 0.1074,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 4.59375,
      "rewards/rejected": -6.3125,
      "step": 480
    },
    {
      "epoch": 1.5705128205128205,
      "grad_norm": 14.674518317046845,
      "learning_rate": 2.648456057007126e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.5390625,
      "logps/chosen": -139.0,
      "logps/rejected": -233.0,
      "loss": 0.1003,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.3125,
      "rewards/margins": 4.9375,
      "rewards/rejected": -6.28125,
      "step": 490
    },
    {
      "epoch": 1.6025641025641026,
      "grad_norm": 18.731133870495857,
      "learning_rate": 2.589073634204275e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.5625,
      "logps/chosen": -144.0,
      "logps/rejected": -251.0,
      "loss": 0.1228,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.109375,
      "rewards/margins": 5.25,
      "rewards/rejected": -6.375,
      "step": 500
    },
    {
      "epoch": 1.6346153846153846,
      "grad_norm": 16.81631331394118,
      "learning_rate": 2.529691211401425e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.5,
      "logps/chosen": -131.0,
      "logps/rejected": -230.0,
      "loss": 0.0819,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.8359375,
      "rewards/margins": 5.375,
      "rewards/rejected": -6.21875,
      "step": 510
    },
    {
      "epoch": 1.6666666666666665,
      "grad_norm": 15.111001669085608,
      "learning_rate": 2.4703087885985747e-07,
      "logits/chosen": -1.546875,
      "logits/rejected": -1.4921875,
      "logps/chosen": -114.0,
      "logps/rejected": -202.0,
      "loss": 0.1091,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.87109375,
      "rewards/margins": 4.78125,
      "rewards/rejected": -5.625,
      "step": 520
    },
    {
      "epoch": 1.6987179487179487,
      "grad_norm": 26.75975769912583,
      "learning_rate": 2.410926365795724e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.515625,
      "logps/chosen": -140.0,
      "logps/rejected": -221.0,
      "loss": 0.1139,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.0625,
      "rewards/margins": 4.8125,
      "rewards/rejected": -5.875,
      "step": 530
    },
    {
      "epoch": 1.7307692307692308,
      "grad_norm": 13.759559493772942,
      "learning_rate": 2.351543942992874e-07,
      "logits/chosen": -1.578125,
      "logits/rejected": -1.5,
      "logps/chosen": -143.0,
      "logps/rejected": -229.0,
      "loss": 0.0978,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.9765625,
      "rewards/margins": 5.09375,
      "rewards/rejected": -6.0625,
      "step": 540
    },
    {
      "epoch": 1.7628205128205128,
      "grad_norm": 18.98900148314797,
      "learning_rate": 2.2921615201900234e-07,
      "logits/chosen": -1.4921875,
      "logits/rejected": -1.5,
      "logps/chosen": -114.0,
      "logps/rejected": -232.0,
      "loss": 0.1103,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.6953125,
      "rewards/margins": 5.03125,
      "rewards/rejected": -5.71875,
      "step": 550
    },
    {
      "epoch": 1.7948717948717947,
      "grad_norm": 7.478935072745169,
      "learning_rate": 2.2327790973871732e-07,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.4375,
      "logps/chosen": -112.0,
      "logps/rejected": -177.0,
      "loss": 0.1139,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6640625,
      "rewards/margins": 4.5625,
      "rewards/rejected": -5.21875,
      "step": 560
    },
    {
      "epoch": 1.8269230769230769,
      "grad_norm": 13.728806584543696,
      "learning_rate": 2.173396674584323e-07,
      "logits/chosen": -1.546875,
      "logits/rejected": -1.5546875,
      "logps/chosen": -130.0,
      "logps/rejected": -239.0,
      "loss": 0.1355,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.466796875,
      "rewards/margins": 4.875,
      "rewards/rejected": -5.34375,
      "step": 570
    },
    {
      "epoch": 1.858974358974359,
      "grad_norm": 9.926742756220793,
      "learning_rate": 2.1140142517814726e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.5078125,
      "logps/chosen": -158.0,
      "logps/rejected": -202.0,
      "loss": 0.1118,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.09375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -5.3125,
      "step": 580
    },
    {
      "epoch": 1.891025641025641,
      "grad_norm": 12.431864869622972,
      "learning_rate": 2.0546318289786222e-07,
      "logits/chosen": -1.5546875,
      "logits/rejected": -1.53125,
      "logps/chosen": -122.0,
      "logps/rejected": -202.0,
      "loss": 0.1325,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.61328125,
      "rewards/margins": 4.65625,
      "rewards/rejected": -5.28125,
      "step": 590
    },
    {
      "epoch": 1.9230769230769231,
      "grad_norm": 12.291656293105886,
      "learning_rate": 1.9952494061757718e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.5625,
      "logps/chosen": -123.0,
      "logps/rejected": -227.0,
      "loss": 0.1085,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.67578125,
      "rewards/margins": 4.8125,
      "rewards/rejected": -5.5,
      "step": 600
    },
    {
      "epoch": 1.9551282051282053,
      "grad_norm": 6.534649119157792,
      "learning_rate": 1.9358669833729216e-07,
      "logits/chosen": -1.5546875,
      "logits/rejected": -1.484375,
      "logps/chosen": -122.0,
      "logps/rejected": -202.0,
      "loss": 0.0935,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.56640625,
      "rewards/margins": 5.0,
      "rewards/rejected": -5.59375,
      "step": 610
    },
    {
      "epoch": 1.9871794871794872,
      "grad_norm": 10.91488724265466,
      "learning_rate": 1.876484560570071e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.515625,
      "logps/chosen": -139.0,
      "logps/rejected": -236.0,
      "loss": 0.1071,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.1953125,
      "rewards/margins": 5.34375,
      "rewards/rejected": -6.53125,
      "step": 620
    },
    {
      "epoch": 2.0,
      "eval_logits/chosen": -1.5,
      "eval_logits/rejected": -1.46875,
      "eval_logps/chosen": -188.0,
      "eval_logps/rejected": -206.0,
      "eval_loss": 0.700976550579071,
      "eval_rewards/accuracies": 0.6428571343421936,
      "eval_rewards/chosen": -3.296875,
      "eval_rewards/margins": 1.3046875,
      "eval_rewards/rejected": -4.59375,
      "eval_runtime": 15.0135,
      "eval_samples_per_second": 13.321,
      "eval_steps_per_second": 0.466,
      "step": 624
    },
    {
      "epoch": 2.019230769230769,
      "grad_norm": 2.9708656343687063,
      "learning_rate": 1.8171021377672207e-07,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.5,
      "logps/chosen": -150.0,
      "logps/rejected": -224.0,
      "loss": 0.071,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.98828125,
      "rewards/margins": 5.40625,
      "rewards/rejected": -6.40625,
      "step": 630
    },
    {
      "epoch": 2.051282051282051,
      "grad_norm": 3.1451139864714266,
      "learning_rate": 1.7577197149643706e-07,
      "logits/chosen": -1.5078125,
      "logits/rejected": -1.4609375,
      "logps/chosen": -121.5,
      "logps/rejected": -198.0,
      "loss": 0.0534,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.8203125,
      "rewards/margins": 4.9375,
      "rewards/rejected": -5.75,
      "step": 640
    },
    {
      "epoch": 2.0833333333333335,
      "grad_norm": 7.314670214652841,
      "learning_rate": 1.6983372921615202e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.546875,
      "logps/chosen": -134.0,
      "logps/rejected": -247.0,
      "loss": 0.0546,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.73046875,
      "rewards/margins": 5.84375,
      "rewards/rejected": -6.5625,
      "step": 650
    },
    {
      "epoch": 2.1153846153846154,
      "grad_norm": 6.986967407965111,
      "learning_rate": 1.6389548693586697e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.53125,
      "logps/chosen": -123.5,
      "logps/rejected": -219.0,
      "loss": 0.0485,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9375,
      "rewards/margins": 5.5625,
      "rewards/rejected": -6.53125,
      "step": 660
    },
    {
      "epoch": 2.1474358974358974,
      "grad_norm": 3.152252590941172,
      "learning_rate": 1.5795724465558193e-07,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.53125,
      "logps/chosen": -121.5,
      "logps/rejected": -264.0,
      "loss": 0.0391,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.97265625,
      "rewards/margins": 6.40625,
      "rewards/rejected": -7.375,
      "step": 670
    },
    {
      "epoch": 2.1794871794871793,
      "grad_norm": 5.616933414198294,
      "learning_rate": 1.520190023752969e-07,
      "logits/chosen": -1.5390625,
      "logits/rejected": -1.5078125,
      "logps/chosen": -136.0,
      "logps/rejected": -225.0,
      "loss": 0.0523,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.265625,
      "rewards/margins": 5.4375,
      "rewards/rejected": -6.6875,
      "step": 680
    },
    {
      "epoch": 2.2115384615384617,
      "grad_norm": 4.462570602831943,
      "learning_rate": 1.4608076009501184e-07,
      "logits/chosen": -1.5390625,
      "logits/rejected": -1.53125,
      "logps/chosen": -141.0,
      "logps/rejected": -247.0,
      "loss": 0.0548,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.9921875,
      "rewards/margins": 5.90625,
      "rewards/rejected": -6.90625,
      "step": 690
    },
    {
      "epoch": 2.2435897435897436,
      "grad_norm": 6.786342065093859,
      "learning_rate": 1.4014251781472683e-07,
      "logits/chosen": -1.5546875,
      "logits/rejected": -1.546875,
      "logps/chosen": -115.0,
      "logps/rejected": -243.0,
      "loss": 0.0328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7890625,
      "rewards/margins": 6.25,
      "rewards/rejected": -7.03125,
      "step": 700
    },
    {
      "epoch": 2.2756410256410255,
      "grad_norm": 10.783463457904992,
      "learning_rate": 1.342042755344418e-07,
      "logits/chosen": -1.5390625,
      "logits/rejected": -1.546875,
      "logps/chosen": -149.0,
      "logps/rejected": -266.0,
      "loss": 0.0556,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.203125,
      "rewards/margins": 6.125,
      "rewards/rejected": -7.3125,
      "step": 710
    },
    {
      "epoch": 2.3076923076923075,
      "grad_norm": 9.0679334215508,
      "learning_rate": 1.2826603325415677e-07,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.4765625,
      "logps/chosen": -121.0,
      "logps/rejected": -217.0,
      "loss": 0.0438,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.80078125,
      "rewards/margins": 5.84375,
      "rewards/rejected": -6.65625,
      "step": 720
    },
    {
      "epoch": 2.33974358974359,
      "grad_norm": 2.164285473082669,
      "learning_rate": 1.2232779097387173e-07,
      "logits/chosen": -1.5078125,
      "logits/rejected": -1.5,
      "logps/chosen": -153.0,
      "logps/rejected": -237.0,
      "loss": 0.0414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2421875,
      "rewards/margins": 6.15625,
      "rewards/rejected": -7.375,
      "step": 730
    },
    {
      "epoch": 2.371794871794872,
      "grad_norm": 8.601824238964523,
      "learning_rate": 1.163895486935867e-07,
      "logits/chosen": -1.4921875,
      "logits/rejected": -1.46875,
      "logps/chosen": -148.0,
      "logps/rejected": -238.0,
      "loss": 0.0358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 5.9375,
      "rewards/rejected": -7.625,
      "step": 740
    },
    {
      "epoch": 2.4038461538461537,
      "grad_norm": 11.674320163023708,
      "learning_rate": 1.1045130641330165e-07,
      "logits/chosen": -1.5234375,
      "logits/rejected": -1.5390625,
      "logps/chosen": -133.0,
      "logps/rejected": -258.0,
      "loss": 0.0523,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 6.4375,
      "rewards/rejected": -8.375,
      "step": 750
    },
    {
      "epoch": 2.435897435897436,
      "grad_norm": 4.6040374115250415,
      "learning_rate": 1.0451306413301662e-07,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.5234375,
      "logps/chosen": -115.5,
      "logps/rejected": -258.0,
      "loss": 0.0425,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.984375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -8.375,
      "step": 760
    },
    {
      "epoch": 2.467948717948718,
      "grad_norm": 6.059248925740205,
      "learning_rate": 9.857482185273158e-08,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.515625,
      "logps/chosen": -137.0,
      "logps/rejected": -233.0,
      "loss": 0.037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2265625,
      "rewards/margins": 5.90625,
      "rewards/rejected": -7.125,
      "step": 770
    },
    {
      "epoch": 2.5,
      "grad_norm": 5.569807408041334,
      "learning_rate": 9.263657957244655e-08,
      "logits/chosen": -1.546875,
      "logits/rejected": -1.53125,
      "logps/chosen": -136.0,
      "logps/rejected": -240.0,
      "loss": 0.0383,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.46875,
      "rewards/margins": 6.78125,
      "rewards/rejected": -8.25,
      "step": 780
    },
    {
      "epoch": 2.532051282051282,
      "grad_norm": 9.080566390829592,
      "learning_rate": 8.669833729216151e-08,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.4375,
      "logps/chosen": -138.0,
      "logps/rejected": -233.0,
      "loss": 0.043,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.125,
      "rewards/margins": 6.1875,
      "rewards/rejected": -8.3125,
      "step": 790
    },
    {
      "epoch": 2.564102564102564,
      "grad_norm": 4.722752417362213,
      "learning_rate": 8.076009501187649e-08,
      "logits/chosen": -1.4609375,
      "logits/rejected": -1.515625,
      "logps/chosen": -129.0,
      "logps/rejected": -244.0,
      "loss": 0.043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.78125,
      "rewards/margins": 6.5625,
      "rewards/rejected": -8.375,
      "step": 800
    },
    {
      "epoch": 2.5961538461538463,
      "grad_norm": 8.320380078890436,
      "learning_rate": 7.482185273159145e-08,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.484375,
      "logps/chosen": -152.0,
      "logps/rejected": -234.0,
      "loss": 0.0311,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.734375,
      "rewards/margins": 6.25,
      "rewards/rejected": -7.96875,
      "step": 810
    },
    {
      "epoch": 2.628205128205128,
      "grad_norm": 10.713886798327927,
      "learning_rate": 6.88836104513064e-08,
      "logits/chosen": -1.5078125,
      "logits/rejected": -1.4765625,
      "logps/chosen": -151.0,
      "logps/rejected": -252.0,
      "loss": 0.0368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6875,
      "rewards/margins": 6.25,
      "rewards/rejected": -7.9375,
      "step": 820
    },
    {
      "epoch": 2.66025641025641,
      "grad_norm": 10.956365394123852,
      "learning_rate": 6.294536817102138e-08,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.4375,
      "logps/chosen": -139.0,
      "logps/rejected": -224.0,
      "loss": 0.0434,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 6.15625,
      "rewards/rejected": -7.875,
      "step": 830
    },
    {
      "epoch": 2.6923076923076925,
      "grad_norm": 5.167870965947855,
      "learning_rate": 5.700712589073634e-08,
      "logits/chosen": -1.5546875,
      "logits/rejected": -1.484375,
      "logps/chosen": -155.0,
      "logps/rejected": -286.0,
      "loss": 0.0367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 7.15625,
      "rewards/rejected": -8.5625,
      "step": 840
    },
    {
      "epoch": 2.7243589743589745,
      "grad_norm": 10.357396384526298,
      "learning_rate": 5.10688836104513e-08,
      "logits/chosen": -1.4765625,
      "logits/rejected": -1.5,
      "logps/chosen": -144.0,
      "logps/rejected": -245.0,
      "loss": 0.0381,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.8125,
      "rewards/margins": 6.46875,
      "rewards/rejected": -8.3125,
      "step": 850
    },
    {
      "epoch": 2.7564102564102564,
      "grad_norm": 4.157699399133391,
      "learning_rate": 4.5130641330166267e-08,
      "logits/chosen": -1.46875,
      "logits/rejected": -1.421875,
      "logps/chosen": -126.0,
      "logps/rejected": -239.0,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 6.25,
      "rewards/rejected": -7.8125,
      "step": 860
    },
    {
      "epoch": 2.7884615384615383,
      "grad_norm": 10.34195664309316,
      "learning_rate": 3.919239904988123e-08,
      "logits/chosen": -1.5234375,
      "logits/rejected": -1.5390625,
      "logps/chosen": -148.0,
      "logps/rejected": -268.0,
      "loss": 0.036,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.296875,
      "rewards/margins": 6.5,
      "rewards/rejected": -8.8125,
      "step": 870
    },
    {
      "epoch": 2.8205128205128203,
      "grad_norm": 13.885196758276543,
      "learning_rate": 3.32541567695962e-08,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.4609375,
      "logps/chosen": -139.0,
      "logps/rejected": -237.0,
      "loss": 0.0386,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 6.4375,
      "rewards/rejected": -8.25,
      "step": 880
    },
    {
      "epoch": 2.8525641025641026,
      "grad_norm": 3.298992157382736,
      "learning_rate": 2.7315914489311164e-08,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.484375,
      "logps/chosen": -142.0,
      "logps/rejected": -264.0,
      "loss": 0.0326,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0,
      "rewards/margins": 6.84375,
      "rewards/rejected": -8.875,
      "step": 890
    },
    {
      "epoch": 2.8846153846153846,
      "grad_norm": 9.55680354164709,
      "learning_rate": 2.1377672209026125e-08,
      "logits/chosen": -1.4609375,
      "logits/rejected": -1.4453125,
      "logps/chosen": -124.0,
      "logps/rejected": -247.0,
      "loss": 0.0386,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 7.0,
      "rewards/rejected": -8.8125,
      "step": 900
    },
    {
      "epoch": 2.9166666666666665,
      "grad_norm": 8.674386112287156,
      "learning_rate": 1.5439429928741092e-08,
      "logits/chosen": -1.5078125,
      "logits/rejected": -1.4765625,
      "logps/chosen": -118.0,
      "logps/rejected": -240.0,
      "loss": 0.0517,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 7.0625,
      "rewards/rejected": -8.4375,
      "step": 910
    },
    {
      "epoch": 2.948717948717949,
      "grad_norm": 6.427610044394737,
      "learning_rate": 9.501187648456057e-09,
      "logits/chosen": -1.5,
      "logits/rejected": -1.4765625,
      "logps/chosen": -148.0,
      "logps/rejected": -256.0,
      "loss": 0.0481,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.84375,
      "rewards/margins": 6.375,
      "rewards/rejected": -8.25,
      "step": 920
    },
    {
      "epoch": 2.980769230769231,
      "grad_norm": 20.775958318465445,
      "learning_rate": 3.562945368171021e-09,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.5,
      "logps/chosen": -141.0,
      "logps/rejected": -232.0,
      "loss": 0.0498,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.640625,
      "rewards/margins": 6.0,
      "rewards/rejected": -7.625,
      "step": 930
    },
    {
      "epoch": 3.0,
      "eval_logits/chosen": -1.4921875,
      "eval_logits/rejected": -1.4609375,
      "eval_logps/chosen": -199.0,
      "eval_logps/rejected": -222.0,
      "eval_loss": 0.830273449420929,
      "eval_rewards/accuracies": 0.625,
      "eval_rewards/chosen": -4.4375,
      "eval_rewards/margins": 1.765625,
      "eval_rewards/rejected": -6.1875,
      "eval_runtime": 16.7003,
      "eval_samples_per_second": 11.976,
      "eval_steps_per_second": 0.419,
      "step": 936
    },
    {
      "epoch": 3.0,
      "step": 936,
      "total_flos": 0.0,
      "train_loss": 0.1719489719113733,
      "train_runtime": 6022.2267,
      "train_samples_per_second": 4.969,
      "train_steps_per_second": 0.155
    }
  ],
  "logging_steps": 10,
  "max_steps": 936,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}