{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 250,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004,
      "grad_norm": 90.84001974484835,
      "learning_rate": 1.25e-07,
      "logits/chosen": 0.98046875,
      "logits/rejected": 0.96484375,
      "logps/chosen": -25.75,
      "logps/rejected": -28.625,
      "loss": 3.419,
      "nll_loss": 2.703125,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.056396484375,
      "rewards/margins": 0.06689453125,
      "rewards/rejected": -0.010498046875,
      "step": 1
    },
    {
      "epoch": 0.008,
      "grad_norm": 93.63863607247485,
      "learning_rate": 2.5e-07,
      "logits/chosen": 1.1015625,
      "logits/rejected": 1.078125,
      "logps/chosen": -32.25,
      "logps/rejected": -30.25,
      "loss": 3.6316,
      "nll_loss": 3.359375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0115966796875,
      "rewards/margins": 0.01458740234375,
      "rewards/rejected": -0.003021240234375,
      "step": 2
    },
    {
      "epoch": 0.012,
      "grad_norm": 89.09751114393859,
      "learning_rate": 3.75e-07,
      "logits/chosen": 0.875,
      "logits/rejected": 0.8984375,
      "logps/chosen": -27.5,
      "logps/rejected": -30.5,
      "loss": 3.603,
      "nll_loss": 2.671875,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.024658203125,
      "rewards/margins": -0.03662109375,
      "rewards/rejected": 0.0120849609375,
      "step": 3
    },
    {
      "epoch": 0.016,
      "grad_norm": 93.60471022707792,
      "learning_rate": 5e-07,
      "logits/chosen": 0.94921875,
      "logits/rejected": 0.94140625,
      "logps/chosen": -23.5,
      "logps/rejected": -29.125,
      "loss": 3.7088,
      "nll_loss": 2.296875,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.027587890625,
      "rewards/margins": -0.0021209716796875,
      "rewards/rejected": 0.0296630859375,
      "step": 4
    },
    {
      "epoch": 0.02,
      "grad_norm": 93.20328417318827,
      "learning_rate": 6.249999999999999e-07,
      "logits/chosen": 0.82421875,
      "logits/rejected": 0.87109375,
      "logps/chosen": -24.375,
      "logps/rejected": -30.625,
      "loss": 3.4743,
      "nll_loss": 2.53125,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.051025390625,
      "rewards/margins": 0.0145263671875,
      "rewards/rejected": 0.036376953125,
      "step": 5
    },
    {
      "epoch": 0.024,
      "grad_norm": 91.22976543634373,
      "learning_rate": 7.5e-07,
      "logits/chosen": 0.94140625,
      "logits/rejected": 0.97265625,
      "logps/chosen": -28.125,
      "logps/rejected": -30.375,
      "loss": 3.5928,
      "nll_loss": 2.78125,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.00156402587890625,
      "rewards/margins": -0.03955078125,
      "rewards/rejected": 0.0380859375,
      "step": 6
    },
    {
      "epoch": 0.028,
      "grad_norm": 101.68787999817542,
      "learning_rate": 8.75e-07,
      "logits/chosen": 0.9453125,
      "logits/rejected": 0.9140625,
      "logps/chosen": -21.625,
      "logps/rejected": -28.25,
      "loss": 3.546,
      "nll_loss": 2.125,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.06884765625,
      "rewards/margins": 0.051025390625,
      "rewards/rejected": 0.01806640625,
      "step": 7
    },
    {
      "epoch": 0.032,
      "grad_norm": 86.70077888201412,
      "learning_rate": 1e-06,
      "logits/chosen": 0.875,
      "logits/rejected": 0.87109375,
      "logps/chosen": -24.25,
      "logps/rejected": -26.625,
      "loss": 3.4702,
      "nll_loss": 2.609375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.11181640625,
      "rewards/margins": 0.07763671875,
      "rewards/rejected": 0.033935546875,
      "step": 8
    },
    {
      "epoch": 0.036,
      "grad_norm": 97.99302928138263,
      "learning_rate": 9.999578688879084e-07,
      "logits/chosen": 0.98046875,
      "logits/rejected": 0.98828125,
      "logps/chosen": -25.0,
      "logps/rejected": -30.625,
      "loss": 3.7406,
      "nll_loss": 2.4375,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.10595703125,
      "rewards/margins": 0.004241943359375,
      "rewards/rejected": 0.10205078125,
      "step": 9
    },
    {
      "epoch": 0.04,
      "grad_norm": 81.98951839046792,
      "learning_rate": 9.998314826517563e-07,
      "logits/chosen": 0.9921875,
      "logits/rejected": 0.99609375,
      "logps/chosen": -22.125,
      "logps/rejected": -25.0,
      "loss": 3.3388,
      "nll_loss": 2.21875,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.201171875,
      "rewards/margins": 0.0306396484375,
      "rewards/rejected": 0.1708984375,
      "step": 10
    },
    {
      "epoch": 0.044,
      "grad_norm": 76.29651704091002,
      "learning_rate": 9.99620862590714e-07,
      "logits/chosen": 0.94921875,
      "logits/rejected": 0.9375,
      "logps/chosen": -21.375,
      "logps/rejected": -25.75,
      "loss": 3.3012,
      "nll_loss": 2.1875,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.27734375,
      "rewards/margins": -0.040771484375,
      "rewards/rejected": 0.318359375,
      "step": 11
    },
    {
      "epoch": 0.048,
      "grad_norm": 81.7871405369469,
      "learning_rate": 9.993260441994114e-07,
      "logits/chosen": 0.83984375,
      "logits/rejected": 0.83984375,
      "logps/chosen": -21.875,
      "logps/rejected": -22.125,
      "loss": 3.2064,
      "nll_loss": 2.296875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.3359375,
      "rewards/margins": 0.0595703125,
      "rewards/rejected": 0.275390625,
      "step": 12
    },
    {
      "epoch": 0.052,
      "grad_norm": 103.64982635961991,
      "learning_rate": 9.989470771619553e-07,
      "logits/chosen": 0.9921875,
      "logits/rejected": 1.015625,
      "logps/chosen": -26.625,
      "logps/rejected": -24.5,
      "loss": 3.3844,
      "nll_loss": 2.671875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.451171875,
      "rewards/margins": 0.11865234375,
      "rewards/rejected": 0.333984375,
      "step": 13
    },
    {
      "epoch": 0.056,
      "grad_norm": 42.34708755561702,
      "learning_rate": 9.984840253435568e-07,
      "logits/chosen": 0.9296875,
      "logits/rejected": 1.0234375,
      "logps/chosen": -18.375,
      "logps/rejected": -19.0,
      "loss": 2.7424,
      "nll_loss": 1.828125,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.7109375,
      "rewards/margins": 0.08544921875,
      "rewards/rejected": 0.625,
      "step": 14
    },
    {
      "epoch": 0.06,
      "grad_norm": 37.96207461928269,
      "learning_rate": 9.979369667797674e-07,
      "logits/chosen": 0.98046875,
      "logits/rejected": 0.95703125,
      "logps/chosen": -16.625,
      "logps/rejected": -22.625,
      "loss": 2.7065,
      "nll_loss": 1.625,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.74609375,
      "rewards/margins": -0.111328125,
      "rewards/rejected": 0.85546875,
      "step": 15
    },
    {
      "epoch": 0.064,
      "grad_norm": 35.96184288602604,
      "learning_rate": 9.973059936633306e-07,
      "logits/chosen": 0.9609375,
      "logits/rejected": 0.97265625,
      "logps/chosen": -15.25,
      "logps/rejected": -22.0,
      "loss": 2.7223,
      "nll_loss": 1.515625,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": 0.6875,
      "rewards/margins": -0.1298828125,
      "rewards/rejected": 0.81640625,
      "step": 16
    },
    {
      "epoch": 0.068,
      "grad_norm": 38.480584517300954,
      "learning_rate": 9.965912123286424e-07,
      "logits/chosen": 0.9921875,
      "logits/rejected": 1.015625,
      "logps/chosen": -20.625,
      "logps/rejected": -23.5,
      "loss": 2.7521,
      "nll_loss": 2.015625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 1.0,
      "rewards/margins": 0.3046875,
      "rewards/rejected": 0.69921875,
      "step": 17
    },
    {
      "epoch": 0.072,
      "grad_norm": 42.985647717347646,
      "learning_rate": 9.95792743233833e-07,
      "logits/chosen": 0.9296875,
      "logits/rejected": 0.96484375,
      "logps/chosen": -17.75,
      "logps/rejected": -22.25,
      "loss": 2.6274,
      "nll_loss": 1.875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.890625,
      "rewards/margins": 0.0089111328125,
      "rewards/rejected": 0.8828125,
      "step": 18
    },
    {
      "epoch": 0.076,
      "grad_norm": 35.66016932258093,
      "learning_rate": 9.949107209404663e-07,
      "logits/chosen": 0.984375,
      "logits/rejected": 1.0078125,
      "logps/chosen": -17.0,
      "logps/rejected": -20.0,
      "loss": 2.5359,
      "nll_loss": 1.734375,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.9921875,
      "rewards/margins": 0.0654296875,
      "rewards/rejected": 0.92578125,
      "step": 19
    },
    {
      "epoch": 0.08,
      "grad_norm": 32.78942676966944,
      "learning_rate": 9.939452940908626e-07,
      "logits/chosen": 0.93359375,
      "logits/rejected": 0.9140625,
      "logps/chosen": -19.375,
      "logps/rejected": -20.75,
      "loss": 2.583,
      "nll_loss": 1.921875,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 1.0703125,
      "rewards/margins": 0.03466796875,
      "rewards/rejected": 1.03125,
      "step": 20
    },
    {
      "epoch": 0.084,
      "grad_norm": 30.390224965176014,
      "learning_rate": 9.92896625383049e-07,
      "logits/chosen": 0.9609375,
      "logits/rejected": 0.97265625,
      "logps/chosen": -16.0,
      "logps/rejected": -20.0,
      "loss": 2.4877,
      "nll_loss": 1.65625,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 1.140625,
      "rewards/margins": 0.0869140625,
      "rewards/rejected": 1.046875,
      "step": 21
    },
    {
      "epoch": 0.088,
      "grad_norm": 26.189318520687596,
      "learning_rate": 9.917648915433411e-07,
      "logits/chosen": 1.0234375,
      "logits/rejected": 1.015625,
      "logps/chosen": -14.75,
      "logps/rejected": -16.625,
      "loss": 2.3129,
      "nll_loss": 1.4296875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.296875,
      "rewards/margins": 0.31640625,
      "rewards/rejected": 0.984375,
      "step": 22
    },
    {
      "epoch": 0.092,
      "grad_norm": 22.39496682670829,
      "learning_rate": 9.905502832965603e-07,
      "logits/chosen": 1.0234375,
      "logits/rejected": 1.015625,
      "logps/chosen": -14.6875,
      "logps/rejected": -17.5,
      "loss": 2.2336,
      "nll_loss": 1.4609375,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 1.2421875,
      "rewards/margins": 0.07958984375,
      "rewards/rejected": 1.1640625,
      "step": 23
    },
    {
      "epoch": 0.096,
      "grad_norm": 22.358547425083277,
      "learning_rate": 9.892530053338907e-07,
      "logits/chosen": 1.078125,
      "logits/rejected": 1.0625,
      "logps/chosen": -15.0625,
      "logps/rejected": -20.625,
      "loss": 2.2508,
      "nll_loss": 1.53125,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 1.1875,
      "rewards/margins": -0.14453125,
      "rewards/rejected": 1.3359375,
      "step": 24
    },
    {
      "epoch": 0.1,
      "grad_norm": 25.286222055847446,
      "learning_rate": 9.87873276278386e-07,
      "logits/chosen": 1.046875,
      "logits/rejected": 1.0234375,
      "logps/chosen": -13.9375,
      "logps/rejected": -17.125,
      "loss": 2.2224,
      "nll_loss": 1.3828125,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 1.421875,
      "rewards/margins": 0.3125,
      "rewards/rejected": 1.109375,
      "step": 25
    },
    {
      "epoch": 0.104,
      "grad_norm": 22.227874316855747,
      "learning_rate": 9.864113286481236e-07,
      "logits/chosen": 1.0,
      "logits/rejected": 1.0234375,
      "logps/chosen": -12.875,
      "logps/rejected": -18.75,
      "loss": 2.1859,
      "nll_loss": 1.3515625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.4609375,
      "rewards/margins": 0.263671875,
      "rewards/rejected": 1.1953125,
      "step": 26
    },
    {
      "epoch": 0.108,
      "grad_norm": 20.247096321249927,
      "learning_rate": 9.84867408817022e-07,
      "logits/chosen": 1.046875,
      "logits/rejected": 1.0703125,
      "logps/chosen": -12.5,
      "logps/rejected": -16.375,
      "loss": 2.0957,
      "nll_loss": 1.265625,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 1.4375,
      "rewards/margins": 0.26953125,
      "rewards/rejected": 1.1640625,
      "step": 27
    },
    {
      "epoch": 0.112,
      "grad_norm": 19.436858660866225,
      "learning_rate": 9.832417769733185e-07,
      "logits/chosen": 1.0703125,
      "logits/rejected": 1.0859375,
      "logps/chosen": -12.25,
      "logps/rejected": -16.625,
      "loss": 2.1721,
      "nll_loss": 1.1640625,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 1.484375,
      "rewards/margins": 0.1494140625,
      "rewards/rejected": 1.3359375,
      "step": 28
    },
    {
      "epoch": 0.116,
      "grad_norm": 18.047996057799406,
      "learning_rate": 9.815347070757231e-07,
      "logits/chosen": 1.015625,
      "logits/rejected": 1.0,
      "logps/chosen": -14.3125,
      "logps/rejected": -16.875,
      "loss": 2.1233,
      "nll_loss": 1.3203125,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 1.8515625,
      "rewards/margins": 0.283203125,
      "rewards/rejected": 1.5703125,
      "step": 29
    },
    {
      "epoch": 0.12,
      "grad_norm": 18.07514136915611,
      "learning_rate": 9.797464868072486e-07,
      "logits/chosen": 1.0625,
      "logits/rejected": 1.109375,
      "logps/chosen": -12.5,
      "logps/rejected": -16.875,
      "loss": 2.1027,
      "nll_loss": 1.2421875,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 1.390625,
      "rewards/margins": 0.22265625,
      "rewards/rejected": 1.171875,
      "step": 30
    },
    {
      "epoch": 0.124,
      "grad_norm": 16.76574433996455,
      "learning_rate": 9.778774175267295e-07,
      "logits/chosen": 1.0390625,
      "logits/rejected": 1.0546875,
      "logps/chosen": -11.125,
      "logps/rejected": -17.625,
      "loss": 1.9701,
      "nll_loss": 1.125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 1.5546875,
      "rewards/margins": 0.296875,
      "rewards/rejected": 1.2578125,
      "step": 31
    },
    {
      "epoch": 0.128,
      "grad_norm": 18.753306438627558,
      "learning_rate": 9.759278142180347e-07,
      "logits/chosen": 1.0546875,
      "logits/rejected": 1.0703125,
      "logps/chosen": -10.125,
      "logps/rejected": -15.4375,
      "loss": 1.9336,
      "nll_loss": 1.0078125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.15625,
      "rewards/margins": 0.150390625,
      "rewards/rejected": 1.0078125,
      "step": 32
    },
    {
      "epoch": 0.132,
      "grad_norm": 15.50417437616818,
      "learning_rate": 9.73898005436987e-07,
      "logits/chosen": 1.03125,
      "logits/rejected": 1.03125,
      "logps/chosen": -11.0625,
      "logps/rejected": -14.75,
      "loss": 1.9432,
      "nll_loss": 1.0703125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.4296875,
      "rewards/margins": 0.333984375,
      "rewards/rejected": 1.09375,
      "step": 33
    },
    {
      "epoch": 0.136,
      "grad_norm": 16.396192118981354,
      "learning_rate": 9.71788333255991e-07,
      "logits/chosen": 1.0390625,
      "logits/rejected": 1.0546875,
      "logps/chosen": -11.5625,
      "logps/rejected": -15.4375,
      "loss": 1.8603,
      "nll_loss": 1.2265625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.796875,
      "rewards/margins": 0.78125,
      "rewards/rejected": 1.015625,
      "step": 34
    },
    {
      "epoch": 0.14,
      "grad_norm": 13.69362502638984,
      "learning_rate": 9.695991532063874e-07,
      "logits/chosen": 1.0390625,
      "logits/rejected": 1.0625,
      "logps/chosen": -11.125,
      "logps/rejected": -14.875,
      "loss": 1.9364,
      "nll_loss": 1.1015625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.90625,
      "rewards/margins": 0.380859375,
      "rewards/rejected": 1.5234375,
      "step": 35
    },
    {
      "epoch": 0.144,
      "grad_norm": 13.536022508840675,
      "learning_rate": 9.673308342185365e-07,
      "logits/chosen": 1.0703125,
      "logits/rejected": 1.0859375,
      "logps/chosen": -11.5625,
      "logps/rejected": -15.625,
      "loss": 1.8253,
      "nll_loss": 1.15625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 1.78125,
      "rewards/margins": 0.3359375,
      "rewards/rejected": 1.4453125,
      "step": 36
    },
    {
      "epoch": 0.148,
      "grad_norm": 15.477743522256894,
      "learning_rate": 9.649837585596444e-07,
      "logits/chosen": 1.125,
      "logits/rejected": 1.0859375,
      "logps/chosen": -9.5625,
      "logps/rejected": -16.375,
      "loss": 1.7562,
      "nll_loss": 0.88671875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.8671875,
      "rewards/margins": 0.5078125,
      "rewards/rejected": 1.359375,
      "step": 37
    },
    {
      "epoch": 0.152,
      "grad_norm": 15.139542721455184,
      "learning_rate": 9.625583217693418e-07,
      "logits/chosen": 1.1171875,
      "logits/rejected": 1.125,
      "logps/chosen": -10.75,
      "logps/rejected": -14.625,
      "loss": 1.8019,
      "nll_loss": 1.0234375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.265625,
      "rewards/margins": 0.83984375,
      "rewards/rejected": 1.4296875,
      "step": 38
    },
    {
      "epoch": 0.156,
      "grad_norm": 17.039592344946968,
      "learning_rate": 9.600549325930259e-07,
      "logits/chosen": 1.1953125,
      "logits/rejected": 1.1953125,
      "logps/chosen": -10.6875,
      "logps/rejected": -16.125,
      "loss": 1.7975,
      "nll_loss": 1.078125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.9921875,
      "rewards/margins": 0.37890625,
      "rewards/rejected": 1.609375,
      "step": 39
    },
    {
      "epoch": 0.16,
      "grad_norm": 11.945267822896529,
      "learning_rate": 9.574740129129765e-07,
      "logits/chosen": 1.1953125,
      "logits/rejected": 1.1875,
      "logps/chosen": -10.5625,
      "logps/rejected": -14.125,
      "loss": 1.6304,
      "nll_loss": 1.0078125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.796875,
      "rewards/margins": 0.482421875,
      "rewards/rejected": 1.3125,
      "step": 40
    },
    {
      "epoch": 0.164,
      "grad_norm": 13.26304188910449,
      "learning_rate": 9.548159976772592e-07,
      "logits/chosen": 1.171875,
      "logits/rejected": 1.1796875,
      "logps/chosen": -10.3125,
      "logps/rejected": -15.125,
      "loss": 1.7527,
      "nll_loss": 1.03125,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 1.8359375,
      "rewards/margins": 0.46875,
      "rewards/rejected": 1.3671875,
      "step": 41
    },
    {
      "epoch": 0.168,
      "grad_norm": 13.145142193151658,
      "learning_rate": 9.520813348264251e-07,
      "logits/chosen": 1.1953125,
      "logits/rejected": 1.2109375,
      "logps/chosen": -9.25,
      "logps/rejected": -19.75,
      "loss": 1.6491,
      "nll_loss": 0.8828125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.109375,
      "rewards/margins": 0.859375,
      "rewards/rejected": 1.25,
      "step": 42
    },
    {
      "epoch": 0.172,
      "grad_norm": 14.23868046028076,
      "learning_rate": 9.492704852180226e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.171875,
      "logps/chosen": -8.75,
      "logps/rejected": -16.0,
      "loss": 1.6864,
      "nll_loss": 0.796875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.8671875,
      "rewards/margins": 0.7578125,
      "rewards/rejected": 1.109375,
      "step": 43
    },
    {
      "epoch": 0.176,
      "grad_norm": 13.971650217328486,
      "learning_rate": 9.463839225489318e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.1796875,
      "logps/chosen": -10.0625,
      "logps/rejected": -15.125,
      "loss": 1.6841,
      "nll_loss": 1.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.046875,
      "rewards/margins": 0.83203125,
      "rewards/rejected": 1.21875,
      "step": 44
    },
    {
      "epoch": 0.18,
      "grad_norm": 12.015757364604882,
      "learning_rate": 9.434221332755338e-07,
      "logits/chosen": 1.1640625,
      "logits/rejected": 1.15625,
      "logps/chosen": -9.375,
      "logps/rejected": -11.75,
      "loss": 1.6724,
      "nll_loss": 0.9375,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 1.921875,
      "rewards/margins": 0.134765625,
      "rewards/rejected": 1.7890625,
      "step": 45
    },
    {
      "epoch": 0.184,
      "grad_norm": 11.903742021506595,
      "learning_rate": 9.403856165317321e-07,
      "logits/chosen": 1.1640625,
      "logits/rejected": 1.1953125,
      "logps/chosen": -8.75,
      "logps/rejected": -19.0,
      "loss": 1.546,
      "nll_loss": 0.8671875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.7578125,
      "rewards/margins": 0.60546875,
      "rewards/rejected": 1.15625,
      "step": 46
    },
    {
      "epoch": 0.188,
      "grad_norm": 11.460580941237025,
      "learning_rate": 9.372748840448361e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.2421875,
      "logps/chosen": -6.375,
      "logps/rejected": -17.375,
      "loss": 1.5881,
      "nll_loss": 0.65234375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.8046875,
      "rewards/margins": 0.46875,
      "rewards/rejected": 1.3359375,
      "step": 47
    },
    {
      "epoch": 0.192,
      "grad_norm": 12.66271092485468,
      "learning_rate": 9.340904600493219e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.1953125,
      "logps/chosen": -7.65625,
      "logps/rejected": -17.25,
      "loss": 1.583,
      "nll_loss": 0.828125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.8984375,
      "rewards/margins": 0.53125,
      "rewards/rejected": 1.3671875,
      "step": 48
    },
    {
      "epoch": 0.196,
      "grad_norm": 14.901259679926502,
      "learning_rate": 9.308328811984869e-07,
      "logits/chosen": 1.171875,
      "logits/rejected": 1.2265625,
      "logps/chosen": -7.90625,
      "logps/rejected": -18.375,
      "loss": 1.4589,
      "nll_loss": 0.7421875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.875,
      "rewards/margins": 0.828125,
      "rewards/rejected": 1.046875,
      "step": 49
    },
    {
      "epoch": 0.2,
      "grad_norm": 11.737584663990637,
      "learning_rate": 9.275026964740101e-07,
      "logits/chosen": 1.140625,
      "logits/rejected": 1.1171875,
      "logps/chosen": -8.125,
      "logps/rejected": -13.1875,
      "loss": 1.4422,
      "nll_loss": 0.7578125,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 1.96875,
      "rewards/margins": 0.5390625,
      "rewards/rejected": 1.4296875,
      "step": 50
    },
    {
      "epoch": 0.204,
      "grad_norm": 10.465507566086854,
      "learning_rate": 9.241004670934347e-07,
      "logits/chosen": 1.1328125,
      "logits/rejected": 1.1484375,
      "logps/chosen": -7.25,
      "logps/rejected": -17.875,
      "loss": 1.5697,
      "nll_loss": 0.671875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.6875,
      "rewards/margins": 0.5859375,
      "rewards/rejected": 1.1015625,
      "step": 51
    },
    {
      "epoch": 0.208,
      "grad_norm": 11.645713255347133,
      "learning_rate": 9.206267664155906e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.1875,
      "logps/chosen": -9.125,
      "logps/rejected": -16.0,
      "loss": 1.5648,
      "nll_loss": 0.94140625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.109375,
      "rewards/margins": 0.8046875,
      "rewards/rejected": 1.3046875,
      "step": 52
    },
    {
      "epoch": 0.212,
      "grad_norm": 12.122656566323922,
      "learning_rate": 9.170821798439683e-07,
      "logits/chosen": 1.15625,
      "logits/rejected": 1.171875,
      "logps/chosen": -6.90625,
      "logps/rejected": -17.125,
      "loss": 1.525,
      "nll_loss": 0.7109375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.8828125,
      "rewards/margins": 0.74609375,
      "rewards/rejected": 1.1328125,
      "step": 53
    },
    {
      "epoch": 0.216,
      "grad_norm": 10.904740492194488,
      "learning_rate": 9.134673047280643e-07,
      "logits/chosen": 1.140625,
      "logits/rejected": 1.171875,
      "logps/chosen": -7.375,
      "logps/rejected": -11.5,
      "loss": 1.5872,
      "nll_loss": 0.6796875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.6953125,
      "rewards/margins": 0.31640625,
      "rewards/rejected": 1.375,
      "step": 54
    },
    {
      "epoch": 0.22,
      "grad_norm": 11.94244354707238,
      "learning_rate": 9.097827502627136e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.1953125,
      "logps/chosen": -6.71875,
      "logps/rejected": -13.625,
      "loss": 1.4571,
      "nll_loss": 0.62890625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.203125,
      "rewards/margins": 0.326171875,
      "rewards/rejected": 1.875,
      "step": 55
    },
    {
      "epoch": 0.224,
      "grad_norm": 10.898501851960246,
      "learning_rate": 9.06029137385425e-07,
      "logits/chosen": 1.21875,
      "logits/rejected": 1.171875,
      "logps/chosen": -8.25,
      "logps/rejected": -15.375,
      "loss": 1.4988,
      "nll_loss": 0.79296875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.421875,
      "rewards/margins": 0.65234375,
      "rewards/rejected": 1.7734375,
      "step": 56
    },
    {
      "epoch": 0.228,
      "grad_norm": 11.292375773132523,
      "learning_rate": 9.022070986717378e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.1796875,
      "logps/chosen": -8.8125,
      "logps/rejected": -17.5,
      "loss": 1.5525,
      "nll_loss": 0.828125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.9765625,
      "rewards/margins": 0.74609375,
      "rewards/rejected": 1.234375,
      "step": 57
    },
    {
      "epoch": 0.232,
      "grad_norm": 13.677184544578704,
      "learning_rate": 8.983172782286179e-07,
      "logits/chosen": 1.21875,
      "logits/rejected": 1.21875,
      "logps/chosen": -6.5,
      "logps/rejected": -12.9375,
      "loss": 1.5552,
      "nll_loss": 0.7109375,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 2.515625,
      "rewards/margins": 0.2275390625,
      "rewards/rejected": 2.28125,
      "step": 58
    },
    {
      "epoch": 0.236,
      "grad_norm": 13.654723770150948,
      "learning_rate": 8.943603315859101e-07,
      "logits/chosen": 1.2421875,
      "logits/rejected": 1.2421875,
      "logps/chosen": -6.96875,
      "logps/rejected": -15.375,
      "loss": 1.4327,
      "nll_loss": 0.66015625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.9921875,
      "rewards/margins": 1.0078125,
      "rewards/rejected": 0.984375,
      "step": 59
    },
    {
      "epoch": 0.24,
      "grad_norm": 10.361981541893595,
      "learning_rate": 8.90336925585864e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.2109375,
      "logps/chosen": -5.71875,
      "logps/rejected": -16.5,
      "loss": 1.5013,
      "nll_loss": 0.5859375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.296875,
      "rewards/margins": 0.99609375,
      "rewards/rejected": 1.296875,
      "step": 60
    },
    {
      "epoch": 0.244,
      "grad_norm": 12.834755538872065,
      "learning_rate": 8.862477382707568e-07,
      "logits/chosen": 1.15625,
      "logits/rejected": 1.140625,
      "logps/chosen": -8.5625,
      "logps/rejected": -15.4375,
      "loss": 1.5555,
      "nll_loss": 0.74609375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 0.83984375,
      "rewards/rejected": 1.0703125,
      "step": 61
    },
    {
      "epoch": 0.248,
      "grad_norm": 10.425380312118477,
      "learning_rate": 8.820934587686245e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.21875,
      "logps/chosen": -7.8125,
      "logps/rejected": -15.3125,
      "loss": 1.5102,
      "nll_loss": 0.76171875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.078125,
      "rewards/margins": 0.62890625,
      "rewards/rejected": 1.453125,
      "step": 62
    },
    {
      "epoch": 0.252,
      "grad_norm": 10.316061787295993,
      "learning_rate": 8.778747871771291e-07,
      "logits/chosen": 1.2265625,
      "logits/rejected": 1.265625,
      "logps/chosen": -6.15625,
      "logps/rejected": -16.75,
      "loss": 1.4838,
      "nll_loss": 0.59375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.171875,
      "rewards/margins": 0.75,
      "rewards/rejected": 1.421875,
      "step": 63
    },
    {
      "epoch": 0.256,
      "grad_norm": 11.866868830656644,
      "learning_rate": 8.735924344455732e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.203125,
      "logps/chosen": -7.71875,
      "logps/rejected": -16.25,
      "loss": 1.3914,
      "nll_loss": 0.7578125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.203125,
      "rewards/margins": 0.92578125,
      "rewards/rejected": 1.2734375,
      "step": 64
    },
    {
      "epoch": 0.26,
      "grad_norm": 13.159883371136859,
      "learning_rate": 8.692471222550885e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.1953125,
      "logps/chosen": -8.8125,
      "logps/rejected": -18.5,
      "loss": 1.5373,
      "nll_loss": 0.8359375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.7578125,
      "rewards/margins": 0.94921875,
      "rewards/rejected": 0.80859375,
      "step": 65
    },
    {
      "epoch": 0.264,
      "grad_norm": 10.735331657215488,
      "learning_rate": 8.64839582897015e-07,
      "logits/chosen": 1.234375,
      "logits/rejected": 1.25,
      "logps/chosen": -8.9375,
      "logps/rejected": -12.0,
      "loss": 1.4267,
      "nll_loss": 0.84375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.25,
      "rewards/margins": 0.30859375,
      "rewards/rejected": 1.9453125,
      "step": 66
    },
    {
      "epoch": 0.268,
      "grad_norm": 11.19711245010709,
      "learning_rate": 8.603705591494916e-07,
      "logits/chosen": 1.25,
      "logits/rejected": 1.21875,
      "logps/chosen": -6.84375,
      "logps/rejected": -14.75,
      "loss": 1.357,
      "nll_loss": 0.72265625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.6953125,
      "rewards/margins": 0.58984375,
      "rewards/rejected": 1.1015625,
      "step": 67
    },
    {
      "epoch": 0.272,
      "grad_norm": 11.97550661008315,
      "learning_rate": 8.5584080415228e-07,
      "logits/chosen": 1.2265625,
      "logits/rejected": 1.2265625,
      "logps/chosen": -6.0625,
      "logps/rejected": -11.4375,
      "loss": 1.4243,
      "nll_loss": 0.56640625,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 1.984375,
      "rewards/margins": 0.1904296875,
      "rewards/rejected": 1.796875,
      "step": 68
    },
    {
      "epoch": 0.276,
      "grad_norm": 12.20745761010426,
      "learning_rate": 8.512510812798425e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.2109375,
      "logps/chosen": -7.5,
      "logps/rejected": -13.8125,
      "loss": 1.4222,
      "nll_loss": 0.69921875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.296875,
      "rewards/margins": 0.65625,
      "rewards/rejected": 1.640625,
      "step": 69
    },
    {
      "epoch": 0.28,
      "grad_norm": 12.64765706733198,
      "learning_rate": 8.466021640126945e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.1796875,
      "logps/chosen": -6.34375,
      "logps/rejected": -15.25,
      "loss": 1.4294,
      "nll_loss": 0.58203125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.109375,
      "rewards/margins": 0.76953125,
      "rewards/rejected": 1.3359375,
      "step": 70
    },
    {
      "epoch": 0.284,
      "grad_norm": 12.94971569659866,
      "learning_rate": 8.418948358070535e-07,
      "logits/chosen": 1.1953125,
      "logits/rejected": 1.171875,
      "logps/chosen": -8.1875,
      "logps/rejected": -13.8125,
      "loss": 1.4353,
      "nll_loss": 0.7734375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 2.171875,
      "rewards/margins": 0.59765625,
      "rewards/rejected": 1.5703125,
      "step": 71
    },
    {
      "epoch": 0.288,
      "grad_norm": 12.30530684787801,
      "learning_rate": 8.37129889962809e-07,
      "logits/chosen": 1.21875,
      "logits/rejected": 1.2578125,
      "logps/chosen": -8.875,
      "logps/rejected": -18.875,
      "loss": 1.4423,
      "nll_loss": 0.8046875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.140625,
      "rewards/margins": 1.078125,
      "rewards/rejected": 1.0625,
      "step": 72
    },
    {
      "epoch": 0.292,
      "grad_norm": 11.236244027667452,
      "learning_rate": 8.323081294898307e-07,
      "logits/chosen": 1.171875,
      "logits/rejected": 1.203125,
      "logps/chosen": -5.9375,
      "logps/rejected": -16.5,
      "loss": 1.3335,
      "nll_loss": 0.625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.328125,
      "rewards/margins": 0.921875,
      "rewards/rejected": 1.40625,
      "step": 73
    },
    {
      "epoch": 0.296,
      "grad_norm": 10.369839744968973,
      "learning_rate": 8.274303669726426e-07,
      "logits/chosen": 1.2421875,
      "logits/rejected": 1.2421875,
      "logps/chosen": -6.5625,
      "logps/rejected": -15.375,
      "loss": 1.4023,
      "nll_loss": 0.58984375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.078125,
      "rewards/margins": 0.9453125,
      "rewards/rejected": 1.1328125,
      "step": 74
    },
    {
      "epoch": 0.3,
      "grad_norm": 10.5653636945963,
      "learning_rate": 8.22497424433482e-07,
      "logits/chosen": 1.265625,
      "logits/rejected": 1.3125,
      "logps/chosen": -6.59375,
      "logps/rejected": -14.5,
      "loss": 1.3905,
      "nll_loss": 0.65625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.171875,
      "rewards/margins": 0.8203125,
      "rewards/rejected": 1.359375,
      "step": 75
    },
    {
      "epoch": 0.304,
      "grad_norm": 11.160863929220326,
      "learning_rate": 8.175101331937692e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.2109375,
      "logps/chosen": -6.3125,
      "logps/rejected": -13.6875,
      "loss": 1.3124,
      "nll_loss": 0.6796875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.390625,
      "rewards/margins": 1.0859375,
      "rewards/rejected": 1.296875,
      "step": 76
    },
    {
      "epoch": 0.308,
      "grad_norm": 14.15246433516739,
      "learning_rate": 8.124693337340092e-07,
      "logits/chosen": 1.2421875,
      "logits/rejected": 1.2265625,
      "logps/chosen": -7.5625,
      "logps/rejected": -17.375,
      "loss": 1.4235,
      "nll_loss": 0.78515625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.078125,
      "rewards/margins": 0.86328125,
      "rewards/rejected": 1.21875,
      "step": 77
    },
    {
      "epoch": 0.312,
      "grad_norm": 10.407569832039918,
      "learning_rate": 8.073758755521504e-07,
      "logits/chosen": 1.171875,
      "logits/rejected": 1.171875,
      "logps/chosen": -6.5625,
      "logps/rejected": -17.75,
      "loss": 1.3378,
      "nll_loss": 0.6484375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.125,
      "rewards/margins": 1.0,
      "rewards/rejected": 1.125,
      "step": 78
    },
    {
      "epoch": 0.316,
      "grad_norm": 9.802442992278246,
      "learning_rate": 8.022306170204231e-07,
      "logits/chosen": 1.1328125,
      "logits/rejected": 1.1640625,
      "logps/chosen": -5.84375,
      "logps/rejected": -18.875,
      "loss": 1.4524,
      "nll_loss": 0.5625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.3125,
      "rewards/margins": 1.3515625,
      "rewards/rejected": 0.96484375,
      "step": 79
    },
    {
      "epoch": 0.32,
      "grad_norm": 11.527882089480844,
      "learning_rate": 7.970344252406831e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.2109375,
      "logps/chosen": -5.78125,
      "logps/rejected": -15.9375,
      "loss": 1.4225,
      "nll_loss": 0.5703125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.171875,
      "rewards/margins": 0.74609375,
      "rewards/rejected": 1.421875,
      "step": 80
    },
    {
      "epoch": 0.324,
      "grad_norm": 10.591457439226241,
      "learning_rate": 7.917881758982836e-07,
      "logits/chosen": 1.2265625,
      "logits/rejected": 1.2421875,
      "logps/chosen": -7.0625,
      "logps/rejected": -13.0,
      "loss": 1.3526,
      "nll_loss": 0.67578125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.9375,
      "rewards/margins": 0.494140625,
      "rewards/rejected": 1.4453125,
      "step": 81
    },
    {
      "epoch": 0.328,
      "grad_norm": 12.214948597017989,
      "learning_rate": 7.86492753114501e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.21875,
      "logps/chosen": -6.3125,
      "logps/rejected": -20.375,
      "loss": 1.279,
      "nll_loss": 0.60546875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.03125,
      "rewards/margins": 1.4375,
      "rewards/rejected": 0.5859375,
      "step": 82
    },
    {
      "epoch": 0.332,
      "grad_norm": 9.417779464605982,
      "learning_rate": 7.811490492975389e-07,
      "logits/chosen": 1.2421875,
      "logits/rejected": 1.265625,
      "logps/chosen": -4.96875,
      "logps/rejected": -17.0,
      "loss": 1.2994,
      "nll_loss": 0.5234375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 1.9609375,
      "rewards/margins": 0.77734375,
      "rewards/rejected": 1.1796875,
      "step": 83
    },
    {
      "epoch": 0.336,
      "grad_norm": 10.535703587473606,
      "learning_rate": 7.757579649921353e-07,
      "logits/chosen": 1.171875,
      "logits/rejected": 1.1953125,
      "logps/chosen": -5.125,
      "logps/rejected": -19.75,
      "loss": 1.3724,
      "nll_loss": 0.51171875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.203125,
      "rewards/margins": 0.92578125,
      "rewards/rejected": 1.28125,
      "step": 84
    },
    {
      "epoch": 0.34,
      "grad_norm": 9.31264272586202,
      "learning_rate": 7.703204087277988e-07,
      "logits/chosen": 1.15625,
      "logits/rejected": 1.15625,
      "logps/chosen": -6.1875,
      "logps/rejected": -15.625,
      "loss": 1.2552,
      "nll_loss": 0.58203125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.9609375,
      "rewards/margins": 0.7734375,
      "rewards/rejected": 1.1875,
      "step": 85
    },
    {
      "epoch": 0.344,
      "grad_norm": 9.834771755358236,
      "learning_rate": 7.648372968656993e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.203125,
      "logps/chosen": -6.71875,
      "logps/rejected": -20.375,
      "loss": 1.3302,
      "nll_loss": 0.62109375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.8203125,
      "rewards/margins": 0.96484375,
      "rewards/rejected": 0.85546875,
      "step": 86
    },
    {
      "epoch": 0.348,
      "grad_norm": 10.165771096354893,
      "learning_rate": 7.593095534442386e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.203125,
      "logps/chosen": -5.15625,
      "logps/rejected": -15.25,
      "loss": 1.2375,
      "nll_loss": 0.55859375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.1875,
      "rewards/margins": 0.8984375,
      "rewards/rejected": 1.2890625,
      "step": 87
    },
    {
      "epoch": 0.352,
      "grad_norm": 16.22798280791584,
      "learning_rate": 7.537381100233278e-07,
      "logits/chosen": 1.2421875,
      "logits/rejected": 1.2421875,
      "logps/chosen": -4.84375,
      "logps/rejected": -19.25,
      "loss": 1.3853,
      "nll_loss": 0.4609375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.109375,
      "rewards/margins": 1.2734375,
      "rewards/rejected": 0.83984375,
      "step": 88
    },
    {
      "epoch": 0.356,
      "grad_norm": 10.189762721333606,
      "learning_rate": 7.481239055273958e-07,
      "logits/chosen": 1.3046875,
      "logits/rejected": 1.296875,
      "logps/chosen": -5.0,
      "logps/rejected": -8.75,
      "loss": 1.3135,
      "nll_loss": 0.49609375,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 2.109375,
      "rewards/margins": 0.1474609375,
      "rewards/rejected": 1.953125,
      "step": 89
    },
    {
      "epoch": 0.36,
      "grad_norm": 9.700804030641727,
      "learning_rate": 7.424678860871584e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.265625,
      "logps/chosen": -4.875,
      "logps/rejected": -19.5,
      "loss": 1.3442,
      "nll_loss": 0.478515625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.21875,
      "rewards/margins": 1.6015625,
      "rewards/rejected": 0.6171875,
      "step": 90
    },
    {
      "epoch": 0.364,
      "grad_norm": 9.412262822397452,
      "learning_rate": 7.367710048801715e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.1953125,
      "logps/chosen": -3.96875,
      "logps/rejected": -13.5,
      "loss": 1.3565,
      "nll_loss": 0.39453125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.125,
      "rewards/margins": 0.65234375,
      "rewards/rejected": 1.46875,
      "step": 91
    },
    {
      "epoch": 0.368,
      "grad_norm": 10.893195650456414,
      "learning_rate": 7.31034221970198e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.21875,
      "logps/chosen": -5.8125,
      "logps/rejected": -23.0,
      "loss": 1.3479,
      "nll_loss": 0.6015625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.9375,
      "rewards/margins": 1.2109375,
      "rewards/rejected": 0.7265625,
      "step": 92
    },
    {
      "epoch": 0.372,
      "grad_norm": 10.215336342693359,
      "learning_rate": 7.252585041454133e-07,
      "logits/chosen": 1.1640625,
      "logits/rejected": 1.1875,
      "logps/chosen": -5.09375,
      "logps/rejected": -19.25,
      "loss": 1.3127,
      "nll_loss": 0.51953125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.921875,
      "rewards/margins": 1.078125,
      "rewards/rejected": 0.84375,
      "step": 93
    },
    {
      "epoch": 0.376,
      "grad_norm": 10.852698170529848,
      "learning_rate": 7.194448247554778e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.21875,
      "logps/chosen": -6.28125,
      "logps/rejected": -14.6875,
      "loss": 1.3314,
      "nll_loss": 0.671875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.046875,
      "rewards/margins": 0.65625,
      "rewards/rejected": 1.390625,
      "step": 94
    },
    {
      "epoch": 0.38,
      "grad_norm": 11.160157641355154,
      "learning_rate": 7.135941635475036e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.2265625,
      "logps/chosen": -7.125,
      "logps/rejected": -22.25,
      "loss": 1.3336,
      "nll_loss": 0.63671875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.25,
      "rewards/margins": 1.40625,
      "rewards/rejected": 0.8359375,
      "step": 95
    },
    {
      "epoch": 0.384,
      "grad_norm": 11.83268475024706,
      "learning_rate": 7.077075065009433e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.21875,
      "logps/chosen": -5.96875,
      "logps/rejected": -15.8125,
      "loss": 1.3246,
      "nll_loss": 0.58984375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.9375,
      "rewards/margins": 0.62890625,
      "rewards/rejected": 1.3046875,
      "step": 96
    },
    {
      "epoch": 0.388,
      "grad_norm": 10.1751967830661,
      "learning_rate": 7.017858456614283e-07,
      "logits/chosen": 1.2578125,
      "logits/rejected": 1.234375,
      "logps/chosen": -4.96875,
      "logps/rejected": -19.5,
      "loss": 1.2867,
      "nll_loss": 0.490234375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.296875,
      "rewards/margins": 1.3203125,
      "rewards/rejected": 0.96875,
      "step": 97
    },
    {
      "epoch": 0.392,
      "grad_norm": 10.601998971866884,
      "learning_rate": 6.958301789735852e-07,
      "logits/chosen": 1.15625,
      "logits/rejected": 1.171875,
      "logps/chosen": -4.78125,
      "logps/rejected": -17.0,
      "loss": 1.3506,
      "nll_loss": 0.484375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.6796875,
      "rewards/margins": 0.76953125,
      "rewards/rejected": 0.90625,
      "step": 98
    },
    {
      "epoch": 0.396,
      "grad_norm": 9.779406708019934,
      "learning_rate": 6.898415101128569e-07,
      "logits/chosen": 1.2578125,
      "logits/rejected": 1.28125,
      "logps/chosen": -6.28125,
      "logps/rejected": -13.875,
      "loss": 1.2903,
      "nll_loss": 0.6171875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.203125,
      "rewards/margins": 0.70703125,
      "rewards/rejected": 1.4921875,
      "step": 99
    },
    {
      "epoch": 0.4,
      "grad_norm": 11.255228657596119,
      "learning_rate": 6.8382084831636e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.203125,
      "logps/chosen": -4.4375,
      "logps/rejected": -25.0,
      "loss": 1.2663,
      "nll_loss": 0.408203125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.09375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": 0.400390625,
      "step": 100
    },
    {
      "epoch": 0.404,
      "grad_norm": 9.493234666356846,
      "learning_rate": 6.777692082128024e-07,
      "logits/chosen": 1.234375,
      "logits/rejected": 1.234375,
      "logps/chosen": -6.625,
      "logps/rejected": -19.0,
      "loss": 1.29,
      "nll_loss": 0.60546875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.0625,
      "rewards/margins": 1.1875,
      "rewards/rejected": 0.87890625,
      "step": 101
    },
    {
      "epoch": 0.408,
      "grad_norm": 10.02656201009763,
      "learning_rate": 6.716876096514943e-07,
      "logits/chosen": 1.171875,
      "logits/rejected": 1.1796875,
      "logps/chosen": -3.640625,
      "logps/rejected": -28.625,
      "loss": 1.2926,
      "nll_loss": 0.328125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.765625,
      "rewards/margins": 1.734375,
      "rewards/rejected": 0.03125,
      "step": 102
    },
    {
      "epoch": 0.412,
      "grad_norm": 12.936276111177527,
      "learning_rate": 6.655770775304782e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.203125,
      "logps/chosen": -6.46875,
      "logps/rejected": -10.6875,
      "loss": 1.3398,
      "nll_loss": 0.65234375,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 2.28125,
      "rewards/margins": 0.48046875,
      "rewards/rejected": 1.8046875,
      "step": 103
    },
    {
      "epoch": 0.416,
      "grad_norm": 11.080497578488936,
      "learning_rate": 6.594386416238094e-07,
      "logits/chosen": 1.1953125,
      "logits/rejected": 1.1953125,
      "logps/chosen": -6.9375,
      "logps/rejected": -14.625,
      "loss": 1.2289,
      "nll_loss": 0.65234375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 1.796875,
      "rewards/margins": 0.6875,
      "rewards/rejected": 1.109375,
      "step": 104
    },
    {
      "epoch": 0.42,
      "grad_norm": 9.566982136498279,
      "learning_rate": 6.532733364080125e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.21875,
      "logps/chosen": -5.0625,
      "logps/rejected": -22.25,
      "loss": 1.2164,
      "nll_loss": 0.462890625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.171875,
      "rewards/margins": 1.2421875,
      "rewards/rejected": 0.9296875,
      "step": 105
    },
    {
      "epoch": 0.424,
      "grad_norm": 9.565841106095371,
      "learning_rate": 6.470822008877482e-07,
      "logits/chosen": 1.25,
      "logits/rejected": 1.28125,
      "logps/chosen": -6.28125,
      "logps/rejected": -14.3125,
      "loss": 1.2998,
      "nll_loss": 0.55859375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.328125,
      "rewards/margins": 0.62890625,
      "rewards/rejected": 1.6953125,
      "step": 106
    },
    {
      "epoch": 0.428,
      "grad_norm": 10.706543148334308,
      "learning_rate": 6.408662784207149e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.1796875,
      "logps/chosen": -5.0625,
      "logps/rejected": -21.75,
      "loss": 1.2774,
      "nll_loss": 0.53125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.953125,
      "rewards/margins": 0.95703125,
      "rewards/rejected": 1.0,
      "step": 107
    },
    {
      "epoch": 0.432,
      "grad_norm": 10.606219490157203,
      "learning_rate": 6.346266165418172e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.234375,
      "logps/chosen": -4.96875,
      "logps/rejected": -15.5625,
      "loss": 1.331,
      "nll_loss": 0.498046875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.625,
      "rewards/margins": 0.9765625,
      "rewards/rejected": 1.6484375,
      "step": 108
    },
    {
      "epoch": 0.436,
      "grad_norm": 14.097117309386473,
      "learning_rate": 6.283642667866315e-07,
      "logits/chosen": 1.234375,
      "logits/rejected": 1.234375,
      "logps/chosen": -5.3125,
      "logps/rejected": -18.5,
      "loss": 1.2949,
      "nll_loss": 0.5078125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.859375,
      "rewards/margins": 0.85546875,
      "rewards/rejected": 1.0,
      "step": 109
    },
    {
      "epoch": 0.44,
      "grad_norm": 9.292312467170014,
      "learning_rate": 6.220802845141957e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.2421875,
      "logps/chosen": -6.34375,
      "logps/rejected": -30.0,
      "loss": 1.2458,
      "nll_loss": 0.578125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.203125,
      "rewards/margins": 2.0625,
      "rewards/rejected": 0.146484375,
      "step": 110
    },
    {
      "epoch": 0.444,
      "grad_norm": 11.721210491862369,
      "learning_rate": 6.157757287291556e-07,
      "logits/chosen": 1.21875,
      "logits/rejected": 1.1796875,
      "logps/chosen": -3.96875,
      "logps/rejected": -25.125,
      "loss": 1.199,
      "nll_loss": 0.40625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.015625,
      "rewards/margins": 1.921875,
      "rewards/rejected": 0.10302734375,
      "step": 111
    },
    {
      "epoch": 0.448,
      "grad_norm": 14.194347503608423,
      "learning_rate": 6.094516619032975e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.1953125,
      "logps/chosen": -6.03125,
      "logps/rejected": -20.625,
      "loss": 1.2678,
      "nll_loss": 0.63671875,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 2.015625,
      "rewards/margins": 1.046875,
      "rewards/rejected": 0.96875,
      "step": 112
    },
    {
      "epoch": 0.452,
      "grad_norm": 11.663474005311688,
      "learning_rate": 6.03109149796494e-07,
      "logits/chosen": 1.1953125,
      "logits/rejected": 1.203125,
      "logps/chosen": -4.09375,
      "logps/rejected": -18.625,
      "loss": 1.2308,
      "nll_loss": 0.400390625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.21875,
      "rewards/margins": 1.015625,
      "rewards/rejected": 1.1953125,
      "step": 113
    },
    {
      "epoch": 0.456,
      "grad_norm": 11.73787149228671,
      "learning_rate": 5.967492612770997e-07,
      "logits/chosen": 1.21875,
      "logits/rejected": 1.2578125,
      "logps/chosen": -7.0625,
      "logps/rejected": -22.875,
      "loss": 1.2719,
      "nll_loss": 0.6875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.109375,
      "rewards/margins": 1.4921875,
      "rewards/rejected": 0.61328125,
      "step": 114
    },
    {
      "epoch": 0.46,
      "grad_norm": 18.765562279107698,
      "learning_rate": 5.90373068141819e-07,
      "logits/chosen": 1.2421875,
      "logits/rejected": 1.234375,
      "logps/chosen": -6.65625,
      "logps/rejected": -23.875,
      "loss": 1.2868,
      "nll_loss": 0.64453125,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.0625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": 0.56640625,
      "step": 115
    },
    {
      "epoch": 0.464,
      "grad_norm": 11.835101741684044,
      "learning_rate": 5.839816449350824e-07,
      "logits/chosen": 1.265625,
      "logits/rejected": 1.265625,
      "logps/chosen": -6.3125,
      "logps/rejected": -18.625,
      "loss": 1.2367,
      "nll_loss": 0.625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.40625,
      "rewards/margins": 0.99609375,
      "rewards/rejected": 1.4140625,
      "step": 116
    },
    {
      "epoch": 0.468,
      "grad_norm": 9.913706129948022,
      "learning_rate": 5.775760687679603e-07,
      "logits/chosen": 1.2265625,
      "logits/rejected": 1.25,
      "logps/chosen": -5.75,
      "logps/rejected": -16.375,
      "loss": 1.1554,
      "nll_loss": 0.55859375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.109375,
      "rewards/margins": 1.234375,
      "rewards/rejected": 0.875,
      "step": 117
    },
    {
      "epoch": 0.472,
      "grad_norm": 10.913087492915105,
      "learning_rate": 5.711574191366427e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.21875,
      "logps/chosen": -3.25,
      "logps/rejected": -20.25,
      "loss": 1.3168,
      "nll_loss": 0.333984375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.15625,
      "rewards/margins": 1.140625,
      "rewards/rejected": 1.0234375,
      "step": 118
    },
    {
      "epoch": 0.476,
      "grad_norm": 10.973356270302578,
      "learning_rate": 5.647267777405177e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.21875,
      "logps/chosen": -5.875,
      "logps/rejected": -13.4375,
      "loss": 1.2444,
      "nll_loss": 0.609375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.96875,
      "rewards/margins": 0.57421875,
      "rewards/rejected": 1.3984375,
      "step": 119
    },
    {
      "epoch": 0.48,
      "grad_norm": 10.36977304647522,
      "learning_rate": 5.582852282998795e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.1875,
      "logps/chosen": -6.1875,
      "logps/rejected": -18.625,
      "loss": 1.2106,
      "nll_loss": 0.54296875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.34375,
      "rewards/margins": 0.96484375,
      "rewards/rejected": 1.375,
      "step": 120
    },
    {
      "epoch": 0.484,
      "grad_norm": 10.479478969202331,
      "learning_rate": 5.518338563732944e-07,
      "logits/chosen": 1.15625,
      "logits/rejected": 1.1640625,
      "logps/chosen": -5.21875,
      "logps/rejected": -18.125,
      "loss": 1.2196,
      "nll_loss": 0.55859375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.90625,
      "rewards/margins": 0.875,
      "rewards/rejected": 1.03125,
      "step": 121
    },
    {
      "epoch": 0.488,
      "grad_norm": 12.292060691577667,
      "learning_rate": 5.453737491746572e-07,
      "logits/chosen": 1.1171875,
      "logits/rejected": 1.140625,
      "logps/chosen": -4.46875,
      "logps/rejected": -15.8125,
      "loss": 1.2847,
      "nll_loss": 0.44921875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.4375,
      "rewards/margins": 1.1328125,
      "rewards/rejected": 1.3125,
      "step": 122
    },
    {
      "epoch": 0.492,
      "grad_norm": 13.647850653915034,
      "learning_rate": 5.389059953899698e-07,
      "logits/chosen": 1.109375,
      "logits/rejected": 1.1328125,
      "logps/chosen": -4.75,
      "logps/rejected": -23.0,
      "loss": 1.3749,
      "nll_loss": 0.47265625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.421875,
      "rewards/margins": 1.4453125,
      "rewards/rejected": 0.96875,
      "step": 123
    },
    {
      "epoch": 0.496,
      "grad_norm": 13.449091878369773,
      "learning_rate": 5.324316849938714e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.21875,
      "logps/chosen": -6.28125,
      "logps/rejected": -15.8125,
      "loss": 1.262,
      "nll_loss": 0.56640625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.140625,
      "rewards/margins": 1.125,
      "rewards/rejected": 1.015625,
      "step": 124
    },
    {
      "epoch": 0.5,
      "grad_norm": 12.727260122548977,
      "learning_rate": 5.259519090659498e-07,
      "logits/chosen": 1.1328125,
      "logits/rejected": 1.1015625,
      "logps/chosen": -4.4375,
      "logps/rejected": -17.5,
      "loss": 1.2112,
      "nll_loss": 0.3984375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 2.4375,
      "rewards/margins": 0.8359375,
      "rewards/rejected": 1.609375,
      "step": 125
    },
    {
      "epoch": 0.504,
      "grad_norm": 12.904271672786484,
      "learning_rate": 5.194677596068688e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.1875,
      "logps/chosen": -6.46875,
      "logps/rejected": -14.9375,
      "loss": 1.3043,
      "nll_loss": 0.6171875,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 2.078125,
      "rewards/margins": 0.203125,
      "rewards/rejected": 1.875,
      "step": 126
    },
    {
      "epoch": 0.508,
      "grad_norm": 11.664774077427383,
      "learning_rate": 5.129803293543391e-07,
      "logits/chosen": 1.078125,
      "logits/rejected": 1.109375,
      "logps/chosen": -3.9375,
      "logps/rejected": -23.375,
      "loss": 1.259,
      "nll_loss": 0.373046875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.234375,
      "rewards/margins": 1.6875,
      "rewards/rejected": 0.5390625,
      "step": 127
    },
    {
      "epoch": 0.512,
      "grad_norm": 11.48911637332855,
      "learning_rate": 5.064907115989654e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.171875,
      "logps/chosen": -6.0,
      "logps/rejected": -21.625,
      "loss": 1.2457,
      "nll_loss": 0.54296875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.03125,
      "rewards/margins": 1.515625,
      "rewards/rejected": 0.515625,
      "step": 128
    },
    {
      "epoch": 0.516,
      "grad_norm": 11.221354732561968,
      "learning_rate": 5e-07,
      "logits/chosen": 1.1484375,
      "logits/rejected": 1.1328125,
      "logps/chosen": -5.21875,
      "logps/rejected": -25.375,
      "loss": 1.2597,
      "nll_loss": 0.5078125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.96875,
      "rewards/margins": 1.546875,
      "rewards/rejected": 0.419921875,
      "step": 129
    },
    {
      "epoch": 0.52,
      "grad_norm": 10.330065282564254,
      "learning_rate": 4.935092884010346e-07,
      "logits/chosen": 1.1484375,
      "logits/rejected": 1.125,
      "logps/chosen": -4.875,
      "logps/rejected": -18.125,
      "loss": 1.2075,
      "nll_loss": 0.48046875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.171875,
      "rewards/margins": 0.84375,
      "rewards/rejected": 1.3359375,
      "step": 130
    },
    {
      "epoch": 0.524,
      "grad_norm": 10.96765827131898,
      "learning_rate": 4.870196706456609e-07,
      "logits/chosen": 1.21875,
      "logits/rejected": 1.203125,
      "logps/chosen": -5.0,
      "logps/rejected": -21.5,
      "loss": 1.271,
      "nll_loss": 0.484375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.015625,
      "rewards/margins": 1.046875,
      "rewards/rejected": 0.97265625,
      "step": 131
    },
    {
      "epoch": 0.528,
      "grad_norm": 11.306955835707726,
      "learning_rate": 4.805322403931311e-07,
      "logits/chosen": 1.078125,
      "logits/rejected": 1.125,
      "logps/chosen": -4.53125,
      "logps/rejected": -21.5,
      "loss": 1.2285,
      "nll_loss": 0.412109375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.171875,
      "rewards/margins": 1.703125,
      "rewards/rejected": 0.474609375,
      "step": 132
    },
    {
      "epoch": 0.532,
      "grad_norm": 11.491623837247948,
      "learning_rate": 4.7404809093405017e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.1640625,
      "logps/chosen": -5.3125,
      "logps/rejected": -18.0,
      "loss": 1.2724,
      "nll_loss": 0.44921875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.21875,
      "rewards/margins": 1.078125,
      "rewards/rejected": 1.140625,
      "step": 133
    },
    {
      "epoch": 0.536,
      "grad_norm": 13.318991974643362,
      "learning_rate": 4.675683150061284e-07,
      "logits/chosen": 1.171875,
      "logits/rejected": 1.1953125,
      "logps/chosen": -5.25,
      "logps/rejected": -18.625,
      "loss": 1.3086,
      "nll_loss": 0.515625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.328125,
      "rewards/margins": 0.87890625,
      "rewards/rejected": 1.453125,
      "step": 134
    },
    {
      "epoch": 0.54,
      "grad_norm": 12.79845150607262,
      "learning_rate": 4.6109400461003003e-07,
      "logits/chosen": 1.234375,
      "logits/rejected": 1.203125,
      "logps/chosen": -6.0,
      "logps/rejected": -16.875,
      "loss": 1.2625,
      "nll_loss": 0.515625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.53125,
      "rewards/margins": 1.234375,
      "rewards/rejected": 1.2890625,
      "step": 135
    },
    {
      "epoch": 0.544,
      "grad_norm": 9.182288245306129,
      "learning_rate": 4.5462625082534286e-07,
      "logits/chosen": 1.1015625,
      "logits/rejected": 1.1328125,
      "logps/chosen": -3.890625,
      "logps/rejected": -17.5,
      "loss": 1.1632,
      "nll_loss": 0.39453125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.34375,
      "rewards/margins": 1.6328125,
      "rewards/rejected": 0.71484375,
      "step": 136
    },
    {
      "epoch": 0.548,
      "grad_norm": 13.192726716701998,
      "learning_rate": 4.481661436267057e-07,
      "logits/chosen": 1.1953125,
      "logits/rejected": 1.1875,
      "logps/chosen": -5.46875,
      "logps/rejected": -20.75,
      "loss": 1.3656,
      "nll_loss": 0.498046875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.40625,
      "rewards/margins": 1.65625,
      "rewards/rejected": 0.75390625,
      "step": 137
    },
    {
      "epoch": 0.552,
      "grad_norm": 10.563546313174774,
      "learning_rate": 4.4171477170012043e-07,
      "logits/chosen": 1.2265625,
      "logits/rejected": 1.2421875,
      "logps/chosen": -4.25,
      "logps/rejected": -22.875,
      "loss": 1.2065,
      "nll_loss": 0.390625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.484375,
      "rewards/margins": 1.53125,
      "rewards/rejected": 0.9453125,
      "step": 138
    },
    {
      "epoch": 0.556,
      "grad_norm": 10.127912435160546,
      "learning_rate": 4.352732222594823e-07,
      "logits/chosen": 1.2265625,
      "logits/rejected": 1.25,
      "logps/chosen": -5.59375,
      "logps/rejected": -17.625,
      "loss": 1.2103,
      "nll_loss": 0.5234375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.3125,
      "rewards/margins": 1.1640625,
      "rewards/rejected": 1.140625,
      "step": 139
    },
    {
      "epoch": 0.56,
      "grad_norm": 10.137596180429622,
      "learning_rate": 4.2884258086335745e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.1875,
      "logps/chosen": -4.9375,
      "logps/rejected": -14.125,
      "loss": 1.3043,
      "nll_loss": 0.4375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.421875,
      "rewards/margins": 0.94921875,
      "rewards/rejected": 1.4765625,
      "step": 140
    },
    {
      "epoch": 0.564,
      "grad_norm": 10.569099883454175,
      "learning_rate": 4.224239312320398e-07,
      "logits/chosen": 1.234375,
      "logits/rejected": 1.2734375,
      "logps/chosen": -6.90625,
      "logps/rejected": -23.125,
      "loss": 1.3308,
      "nll_loss": 0.58984375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.7109375,
      "rewards/margins": 1.1875,
      "rewards/rejected": 0.5234375,
      "step": 141
    },
    {
      "epoch": 0.568,
      "grad_norm": 10.854956318015526,
      "learning_rate": 4.160183550649175e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.1875,
      "logps/chosen": -4.46875,
      "logps/rejected": -19.25,
      "loss": 1.2546,
      "nll_loss": 0.455078125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.40625,
      "rewards/margins": 1.3671875,
      "rewards/rejected": 1.03125,
      "step": 142
    },
    {
      "epoch": 0.572,
      "grad_norm": 10.941246000391953,
      "learning_rate": 4.09626931858181e-07,
      "logits/chosen": 1.265625,
      "logits/rejected": 1.265625,
      "logps/chosen": -4.40625,
      "logps/rejected": -13.25,
      "loss": 1.2541,
      "nll_loss": 0.435546875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.203125,
      "rewards/margins": 0.5390625,
      "rewards/rejected": 1.671875,
      "step": 143
    },
    {
      "epoch": 0.576,
      "grad_norm": 11.708985201825207,
      "learning_rate": 4.0325073872290015e-07,
      "logits/chosen": 1.21875,
      "logits/rejected": 1.2421875,
      "logps/chosen": -6.375,
      "logps/rejected": -15.75,
      "loss": 1.2988,
      "nll_loss": 0.57421875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.21875,
      "rewards/margins": 1.03125,
      "rewards/rejected": 1.1953125,
      "step": 144
    },
    {
      "epoch": 0.58,
      "grad_norm": 17.168400775199007,
      "learning_rate": 3.96890850203506e-07,
      "logits/chosen": 1.25,
      "logits/rejected": 1.265625,
      "logps/chosen": -6.0,
      "logps/rejected": -18.375,
      "loss": 1.1609,
      "nll_loss": 0.5703125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.625,
      "rewards/margins": 1.4140625,
      "rewards/rejected": 1.2109375,
      "step": 145
    },
    {
      "epoch": 0.584,
      "grad_norm": 12.001725465676962,
      "learning_rate": 3.9054833809670265e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.2265625,
      "logps/chosen": -5.84375,
      "logps/rejected": -20.5,
      "loss": 1.211,
      "nll_loss": 0.5234375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.125,
      "rewards/margins": 1.328125,
      "rewards/rejected": 0.796875,
      "step": 146
    },
    {
      "epoch": 0.588,
      "grad_norm": 11.340434870524739,
      "learning_rate": 3.8422427127084436e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.203125,
      "logps/chosen": -7.0,
      "logps/rejected": -21.75,
      "loss": 1.2451,
      "nll_loss": 0.6328125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.0625,
      "rewards/margins": 1.1328125,
      "rewards/rejected": 0.9296875,
      "step": 147
    },
    {
      "epoch": 0.592,
      "grad_norm": 9.888995312706117,
      "learning_rate": 3.779197154858044e-07,
      "logits/chosen": 1.1953125,
      "logits/rejected": 1.1953125,
      "logps/chosen": -5.21875,
      "logps/rejected": -17.25,
      "loss": 1.1759,
      "nll_loss": 0.5078125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.53125,
      "rewards/margins": 1.140625,
      "rewards/rejected": 1.390625,
      "step": 148
    },
    {
      "epoch": 0.596,
      "grad_norm": 11.055589823055614,
      "learning_rate": 3.716357332133686e-07,
      "logits/chosen": 1.125,
      "logits/rejected": 1.1640625,
      "logps/chosen": -5.5625,
      "logps/rejected": -21.875,
      "loss": 1.2459,
      "nll_loss": 0.52734375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.109375,
      "rewards/margins": 1.359375,
      "rewards/rejected": 0.75,
      "step": 149
    },
    {
      "epoch": 0.6,
      "grad_norm": 10.860785033516322,
      "learning_rate": 3.653733834581827e-07,
      "logits/chosen": 1.125,
      "logits/rejected": 1.09375,
      "logps/chosen": -4.40625,
      "logps/rejected": -13.625,
      "loss": 1.2623,
      "nll_loss": 0.396484375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 0.9453125,
      "rewards/rejected": 0.96484375,
      "step": 150
    },
    {
      "epoch": 0.604,
      "grad_norm": 10.973234026852442,
      "learning_rate": 3.591337215792851e-07,
      "logits/chosen": 1.125,
      "logits/rejected": 1.1640625,
      "logps/chosen": -4.65625,
      "logps/rejected": -18.0,
      "loss": 1.1942,
      "nll_loss": 0.46875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.203125,
      "rewards/margins": 1.0078125,
      "rewards/rejected": 1.203125,
      "step": 151
    },
    {
      "epoch": 0.608,
      "grad_norm": 10.179285058727595,
      "learning_rate": 3.5291779911225185e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.203125,
      "logps/chosen": -4.46875,
      "logps/rejected": -23.5,
      "loss": 1.1627,
      "nll_loss": 0.431640625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.390625,
      "rewards/margins": 1.6328125,
      "rewards/rejected": 0.7578125,
      "step": 152
    },
    {
      "epoch": 0.612,
      "grad_norm": 10.366061661226649,
      "learning_rate": 3.4672666359198757e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.1875,
      "logps/chosen": -3.796875,
      "logps/rejected": -17.75,
      "loss": 1.2135,
      "nll_loss": 0.412109375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.828125,
      "rewards/margins": 1.5078125,
      "rewards/rejected": 1.3203125,
      "step": 153
    },
    {
      "epoch": 0.616,
      "grad_norm": 11.300580663859082,
      "learning_rate": 3.405613583761907e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.1953125,
      "logps/chosen": -6.4375,
      "logps/rejected": -13.9375,
      "loss": 1.2437,
      "nll_loss": 0.6796875,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 2.203125,
      "rewards/margins": 0.61328125,
      "rewards/rejected": 1.59375,
      "step": 154
    },
    {
      "epoch": 0.62,
      "grad_norm": 11.765886781112657,
      "learning_rate": 3.3442292246952186e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.1796875,
      "logps/chosen": -6.4375,
      "logps/rejected": -25.75,
      "loss": 1.2218,
      "nll_loss": 0.61328125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.734375,
      "rewards/margins": 1.484375,
      "rewards/rejected": 0.248046875,
      "step": 155
    },
    {
      "epoch": 0.624,
      "grad_norm": 12.965990069205224,
      "learning_rate": 3.283123903485059e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.21875,
      "logps/chosen": -5.84375,
      "logps/rejected": -21.0,
      "loss": 1.2737,
      "nll_loss": 0.5703125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.34375,
      "rewards/margins": 1.3125,
      "rewards/rejected": 1.0234375,
      "step": 156
    },
    {
      "epoch": 0.628,
      "grad_norm": 21.892665126768843,
      "learning_rate": 3.222307917871977e-07,
      "logits/chosen": 1.1640625,
      "logits/rejected": 1.21875,
      "logps/chosen": -5.28125,
      "logps/rejected": -24.75,
      "loss": 1.2363,
      "nll_loss": 0.4765625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.0,
      "rewards/margins": 2.046875,
      "rewards/rejected": -0.05517578125,
      "step": 157
    },
    {
      "epoch": 0.632,
      "grad_norm": 11.166231452253136,
      "learning_rate": 3.161791516836399e-07,
      "logits/chosen": 1.109375,
      "logits/rejected": 1.1328125,
      "logps/chosen": -6.84375,
      "logps/rejected": -25.25,
      "loss": 1.2571,
      "nll_loss": 0.60546875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.171875,
      "rewards/margins": 1.453125,
      "rewards/rejected": 0.7109375,
      "step": 158
    },
    {
      "epoch": 0.636,
      "grad_norm": 12.283746566871063,
      "learning_rate": 3.1015848988714307e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.21875,
      "logps/chosen": -6.90625,
      "logps/rejected": -23.5,
      "loss": 1.2295,
      "nll_loss": 0.64453125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.484375,
      "rewards/margins": 1.78125,
      "rewards/rejected": 0.70703125,
      "step": 159
    },
    {
      "epoch": 0.64,
      "grad_norm": 11.997709728929365,
      "learning_rate": 3.0416982102641487e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.1953125,
      "logps/chosen": -5.78125,
      "logps/rejected": -19.5,
      "loss": 1.2439,
      "nll_loss": 0.578125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.40625,
      "rewards/margins": 1.515625,
      "rewards/rejected": 0.8828125,
      "step": 160
    },
    {
      "epoch": 0.644,
      "grad_norm": 12.312941284655942,
      "learning_rate": 2.982141543385717e-07,
      "logits/chosen": 1.1953125,
      "logits/rejected": 1.203125,
      "logps/chosen": -4.5625,
      "logps/rejected": -17.0,
      "loss": 1.2541,
      "nll_loss": 0.44921875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.203125,
      "rewards/margins": 0.85546875,
      "rewards/rejected": 1.34375,
      "step": 161
    },
    {
      "epoch": 0.648,
      "grad_norm": 10.583664987026914,
      "learning_rate": 2.922924934990568e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.2109375,
      "logps/chosen": -4.5,
      "logps/rejected": -19.625,
      "loss": 1.1992,
      "nll_loss": 0.462890625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.046875,
      "rewards/margins": 1.390625,
      "rewards/rejected": 0.65625,
      "step": 162
    },
    {
      "epoch": 0.652,
      "grad_norm": 12.369672652872328,
      "learning_rate": 2.864058364524964e-07,
      "logits/chosen": 1.2578125,
      "logits/rejected": 1.25,
      "logps/chosen": -4.59375,
      "logps/rejected": -17.125,
      "loss": 1.2223,
      "nll_loss": 0.478515625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.296875,
      "rewards/margins": 1.484375,
      "rewards/rejected": 0.8125,
      "step": 163
    },
    {
      "epoch": 0.656,
      "grad_norm": 10.926472150443256,
      "learning_rate": 2.8055517524452215e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.203125,
      "logps/chosen": -5.28125,
      "logps/rejected": -19.0,
      "loss": 1.1861,
      "nll_loss": 0.48046875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.140625,
      "rewards/margins": 1.21875,
      "rewards/rejected": 0.91796875,
      "step": 164
    },
    {
      "epoch": 0.66,
      "grad_norm": 19.45457445267617,
      "learning_rate": 2.7474149585458664e-07,
      "logits/chosen": 1.109375,
      "logits/rejected": 1.109375,
      "logps/chosen": -3.953125,
      "logps/rejected": -26.625,
      "loss": 1.2098,
      "nll_loss": 0.35546875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.890625,
      "rewards/margins": 1.6484375,
      "rewards/rejected": 0.236328125,
      "step": 165
    },
    {
      "epoch": 0.664,
      "grad_norm": 13.019845340147054,
      "learning_rate": 2.6896577802980184e-07,
      "logits/chosen": 1.21875,
      "logits/rejected": 1.21875,
      "logps/chosen": -3.125,
      "logps/rejected": -24.0,
      "loss": 1.2028,
      "nll_loss": 0.291015625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.0625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": 0.375,
      "step": 166
    },
    {
      "epoch": 0.668,
      "grad_norm": 11.31879875355034,
      "learning_rate": 2.632289951198285e-07,
      "logits/chosen": 1.171875,
      "logits/rejected": 1.171875,
      "logps/chosen": -5.28125,
      "logps/rejected": -21.875,
      "loss": 1.2914,
      "nll_loss": 0.482421875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.4375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": 1.0546875,
      "step": 167
    },
    {
      "epoch": 0.672,
      "grad_norm": 14.312221004137449,
      "learning_rate": 2.575321139128417e-07,
      "logits/chosen": 1.2265625,
      "logits/rejected": 1.2734375,
      "logps/chosen": -5.1875,
      "logps/rejected": -20.375,
      "loss": 1.2442,
      "nll_loss": 0.484375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.4375,
      "rewards/margins": 1.3203125,
      "rewards/rejected": 1.109375,
      "step": 168
    },
    {
      "epoch": 0.676,
      "grad_norm": 11.88685392821398,
      "learning_rate": 2.5187609447260417e-07,
      "logits/chosen": 1.1328125,
      "logits/rejected": 1.09375,
      "logps/chosen": -4.4375,
      "logps/rejected": -19.875,
      "loss": 1.1703,
      "nll_loss": 0.453125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.578125,
      "rewards/margins": 1.8828125,
      "rewards/rejected": 0.703125,
      "step": 169
    },
    {
      "epoch": 0.68,
      "grad_norm": 17.314779451265547,
      "learning_rate": 2.462618899766722e-07,
      "logits/chosen": 1.1328125,
      "logits/rejected": 1.1328125,
      "logps/chosen": -6.1875,
      "logps/rejected": -23.625,
      "loss": 1.2089,
      "nll_loss": 0.6015625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.921875,
      "rewards/margins": 1.453125,
      "rewards/rejected": 0.470703125,
      "step": 170
    },
    {
      "epoch": 0.684,
      "grad_norm": 11.85551915443719,
      "learning_rate": 2.406904465557614e-07,
      "logits/chosen": 1.1640625,
      "logits/rejected": 1.1328125,
      "logps/chosen": -5.15625,
      "logps/rejected": -23.0,
      "loss": 1.1893,
      "nll_loss": 0.5078125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.171875,
      "rewards/margins": 1.28125,
      "rewards/rejected": 0.8828125,
      "step": 171
    },
    {
      "epoch": 0.688,
      "grad_norm": 12.228210769141855,
      "learning_rate": 2.3516270313430081e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.1875,
      "logps/chosen": -6.8125,
      "logps/rejected": -20.5,
      "loss": 1.2107,
      "nll_loss": 0.6171875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.328125,
      "rewards/margins": 1.59375,
      "rewards/rejected": 0.734375,
      "step": 172
    },
    {
      "epoch": 0.692,
      "grad_norm": 11.027522143396848,
      "learning_rate": 2.2967959127220137e-07,
      "logits/chosen": 1.1328125,
      "logits/rejected": 1.109375,
      "logps/chosen": -3.84375,
      "logps/rejected": -17.75,
      "loss": 1.1306,
      "nll_loss": 0.365234375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.375,
      "rewards/margins": 1.390625,
      "rewards/rejected": 0.9765625,
      "step": 173
    },
    {
      "epoch": 0.696,
      "grad_norm": 13.079512510371146,
      "learning_rate": 2.242420350078647e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.15625,
      "logps/chosen": -5.625,
      "logps/rejected": -19.125,
      "loss": 1.2236,
      "nll_loss": 0.53125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.453125,
      "rewards/margins": 1.2421875,
      "rewards/rejected": 1.2109375,
      "step": 174
    },
    {
      "epoch": 0.7,
      "grad_norm": 12.476234076652355,
      "learning_rate": 2.1885095070246113e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.1484375,
      "logps/chosen": -5.3125,
      "logps/rejected": -22.125,
      "loss": 1.2699,
      "nll_loss": 0.5,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.3125,
      "rewards/margins": 1.5703125,
      "rewards/rejected": 0.75,
      "step": 175
    },
    {
      "epoch": 0.704,
      "grad_norm": 13.41091780256611,
      "learning_rate": 2.1350724688549904e-07,
      "logits/chosen": 1.0625,
      "logits/rejected": 1.1015625,
      "logps/chosen": -4.40625,
      "logps/rejected": -22.5,
      "loss": 1.2699,
      "nll_loss": 0.421875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 1.890625,
      "rewards/margins": 1.296875,
      "rewards/rejected": 0.59375,
      "step": 176
    },
    {
      "epoch": 0.708,
      "grad_norm": 11.558122765112593,
      "learning_rate": 2.0821182410171633e-07,
      "logits/chosen": 1.1328125,
      "logits/rejected": 1.1171875,
      "logps/chosen": -4.3125,
      "logps/rejected": -19.375,
      "loss": 1.3392,
      "nll_loss": 0.40625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.015625,
      "rewards/margins": 1.1875,
      "rewards/rejected": 0.83203125,
      "step": 177
    },
    {
      "epoch": 0.712,
      "grad_norm": 13.675221151303951,
      "learning_rate": 2.0296557475931687e-07,
      "logits/chosen": 1.1015625,
      "logits/rejected": 1.1171875,
      "logps/chosen": -6.09375,
      "logps/rejected": -17.0,
      "loss": 1.286,
      "nll_loss": 0.58203125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.40625,
      "rewards/margins": 1.015625,
      "rewards/rejected": 1.3984375,
      "step": 178
    },
    {
      "epoch": 0.716,
      "grad_norm": 11.77241428897747,
      "learning_rate": 1.9776938297957686e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.2734375,
      "logps/chosen": -3.75,
      "logps/rejected": -15.8125,
      "loss": 1.2497,
      "nll_loss": 0.400390625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.515625,
      "rewards/margins": 1.125,
      "rewards/rejected": 1.3828125,
      "step": 179
    },
    {
      "epoch": 0.72,
      "grad_norm": 13.110435333261227,
      "learning_rate": 1.9262412444784958e-07,
      "logits/chosen": 1.2890625,
      "logits/rejected": 1.2578125,
      "logps/chosen": -6.53125,
      "logps/rejected": -12.3125,
      "loss": 1.2314,
      "nll_loss": 0.625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.328125,
      "rewards/margins": 0.59375,
      "rewards/rejected": 1.7265625,
      "step": 180
    },
    {
      "epoch": 0.724,
      "grad_norm": 11.650307902708178,
      "learning_rate": 1.8753066626599084e-07,
      "logits/chosen": 1.125,
      "logits/rejected": 1.125,
      "logps/chosen": -4.46875,
      "logps/rejected": -24.0,
      "loss": 1.2924,
      "nll_loss": 0.4609375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.25,
      "rewards/margins": 1.4140625,
      "rewards/rejected": 0.8359375,
      "step": 181
    },
    {
      "epoch": 0.728,
      "grad_norm": 13.580946392431395,
      "learning_rate": 1.8248986680623074e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.2421875,
      "logps/chosen": -5.375,
      "logps/rejected": -16.125,
      "loss": 1.2858,
      "nll_loss": 0.546875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.03125,
      "rewards/margins": 0.703125,
      "rewards/rejected": 1.328125,
      "step": 182
    },
    {
      "epoch": 0.732,
      "grad_norm": 12.225985724922241,
      "learning_rate": 1.7750257556651805e-07,
      "logits/chosen": 1.15625,
      "logits/rejected": 1.171875,
      "logps/chosen": -4.5625,
      "logps/rejected": -22.125,
      "loss": 1.2099,
      "nll_loss": 0.443359375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.046875,
      "rewards/margins": 1.625,
      "rewards/rejected": 0.416015625,
      "step": 183
    },
    {
      "epoch": 0.736,
      "grad_norm": 12.563946894349364,
      "learning_rate": 1.725696330273575e-07,
      "logits/chosen": 1.2265625,
      "logits/rejected": 1.234375,
      "logps/chosen": -4.40625,
      "logps/rejected": -18.875,
      "loss": 1.186,
      "nll_loss": 0.447265625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.421875,
      "rewards/margins": 1.2578125,
      "rewards/rejected": 1.1640625,
      "step": 184
    },
    {
      "epoch": 0.74,
      "grad_norm": 13.255428758141953,
      "learning_rate": 1.6769187051016932e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.1484375,
      "logps/chosen": -4.5,
      "logps/rejected": -17.875,
      "loss": 1.2881,
      "nll_loss": 0.470703125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.21875,
      "rewards/margins": 1.109375,
      "rewards/rejected": 1.109375,
      "step": 185
    },
    {
      "epoch": 0.744,
      "grad_norm": 11.194260302105462,
      "learning_rate": 1.6287011003719102e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.1796875,
      "logps/chosen": -5.09375,
      "logps/rejected": -18.25,
      "loss": 1.1767,
      "nll_loss": 0.5,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.640625,
      "rewards/margins": 1.5859375,
      "rewards/rejected": 1.0625,
      "step": 186
    },
    {
      "epoch": 0.748,
      "grad_norm": 12.148074503850838,
      "learning_rate": 1.581051641929465e-07,
      "logits/chosen": 1.1171875,
      "logits/rejected": 1.15625,
      "logps/chosen": -5.28125,
      "logps/rejected": -16.5,
      "loss": 1.2704,
      "nll_loss": 0.5,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.15625,
      "rewards/margins": 0.98046875,
      "rewards/rejected": 1.171875,
      "step": 187
    },
    {
      "epoch": 0.752,
      "grad_norm": 12.395801551212454,
      "learning_rate": 1.5339783598730565e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.2109375,
      "logps/chosen": -4.71875,
      "logps/rejected": -18.75,
      "loss": 1.2776,
      "nll_loss": 0.458984375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.078125,
      "rewards/margins": 1.296875,
      "rewards/rejected": 0.7734375,
      "step": 188
    },
    {
      "epoch": 0.756,
      "grad_norm": 10.67438569858004,
      "learning_rate": 1.4874891872015734e-07,
      "logits/chosen": 1.2578125,
      "logits/rejected": 1.203125,
      "logps/chosen": -6.125,
      "logps/rejected": -18.75,
      "loss": 1.2794,
      "nll_loss": 0.5546875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.5,
      "rewards/margins": 1.40625,
      "rewards/rejected": 1.09375,
      "step": 189
    },
    {
      "epoch": 0.76,
      "grad_norm": 12.475216208621594,
      "learning_rate": 1.4415919584771995e-07,
      "logits/chosen": 1.2578125,
      "logits/rejected": 1.265625,
      "logps/chosen": -4.75,
      "logps/rejected": -18.625,
      "loss": 1.2209,
      "nll_loss": 0.451171875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.421875,
      "rewards/margins": 1.4765625,
      "rewards/rejected": 0.94921875,
      "step": 190
    },
    {
      "epoch": 0.764,
      "grad_norm": 10.90649391121936,
      "learning_rate": 1.3962944085050832e-07,
      "logits/chosen": 1.234375,
      "logits/rejected": 1.2109375,
      "logps/chosen": -6.90625,
      "logps/rejected": -17.625,
      "loss": 1.2429,
      "nll_loss": 0.625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.125,
      "rewards/margins": 1.0859375,
      "rewards/rejected": 1.0390625,
      "step": 191
    },
    {
      "epoch": 0.768,
      "grad_norm": 12.353299907170381,
      "learning_rate": 1.3516041710298497e-07,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.1953125,
      "logps/chosen": -3.75,
      "logps/rejected": -12.75,
      "loss": 1.137,
      "nll_loss": 0.3828125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.5,
      "rewards/margins": 0.6796875,
      "rewards/rejected": 1.8203125,
      "step": 192
    },
    {
      "epoch": 0.772,
      "grad_norm": 11.398941153389819,
      "learning_rate": 1.3075287774491146e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.1796875,
      "logps/chosen": -3.5625,
      "logps/rejected": -20.25,
      "loss": 1.2329,
      "nll_loss": 0.369140625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.46875,
      "rewards/margins": 1.296875,
      "rewards/rejected": 1.1640625,
      "step": 193
    },
    {
      "epoch": 0.776,
      "grad_norm": 13.85380958569379,
      "learning_rate": 1.2640756555442683e-07,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.2109375,
      "logps/chosen": -6.28125,
      "logps/rejected": -18.25,
      "loss": 1.2127,
      "nll_loss": 0.5703125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.578125,
      "rewards/margins": 1.4140625,
      "rewards/rejected": 1.1640625,
      "step": 194
    },
    {
      "epoch": 0.78,
      "grad_norm": 13.267633587995695,
      "learning_rate": 1.2212521282287093e-07,
      "logits/chosen": 1.21875,
      "logits/rejected": 1.21875,
      "logps/chosen": -5.40625,
      "logps/rejected": -19.625,
      "loss": 1.2959,
      "nll_loss": 0.55078125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.53125,
      "rewards/margins": 1.1640625,
      "rewards/rejected": 1.359375,
      "step": 195
    },
    {
      "epoch": 0.784,
      "grad_norm": 15.787904836631533,
      "learning_rate": 1.179065412313755e-07,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.15625,
      "logps/chosen": -7.03125,
      "logps/rejected": -15.25,
      "loss": 1.3018,
      "nll_loss": 0.69921875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 1.921875,
      "rewards/margins": 0.61328125,
      "rewards/rejected": 1.3125,
      "step": 196
    },
    {
      "epoch": 0.788,
      "grad_norm": 11.642080128952893,
      "learning_rate": 1.137522617292434e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.21875,
      "logps/chosen": -4.71875,
      "logps/rejected": -9.6875,
      "loss": 1.2841,
      "nll_loss": 0.45703125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.546875,
      "rewards/margins": 0.7265625,
      "rewards/rejected": 1.828125,
      "step": 197
    },
    {
      "epoch": 0.792,
      "grad_norm": 19.23607159744212,
      "learning_rate": 1.0966307441413597e-07,
      "logits/chosen": 1.1171875,
      "logits/rejected": 1.140625,
      "logps/chosen": -6.0,
      "logps/rejected": -16.0,
      "loss": 1.2262,
      "nll_loss": 0.6015625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.796875,
      "rewards/margins": 1.3359375,
      "rewards/rejected": 1.4609375,
      "step": 198
    },
    {
      "epoch": 0.796,
      "grad_norm": 14.441184784024456,
      "learning_rate": 1.0563966841408995e-07,
      "logits/chosen": 1.171875,
      "logits/rejected": 1.1875,
      "logps/chosen": -4.9375,
      "logps/rejected": -18.75,
      "loss": 1.1646,
      "nll_loss": 0.486328125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.5625,
      "rewards/margins": 1.3671875,
      "rewards/rejected": 1.203125,
      "step": 199
    },
    {
      "epoch": 0.8,
      "grad_norm": 12.165181869834347,
      "learning_rate": 1.0168272177138199e-07,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.2265625,
      "logps/chosen": -5.625,
      "logps/rejected": -21.25,
      "loss": 1.2264,
      "nll_loss": 0.53515625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.328125,
      "rewards/margins": 1.2421875,
      "rewards/rejected": 1.0859375,
      "step": 200
    },
    {
      "epoch": 0.804,
      "grad_norm": 12.286496963943826,
      "learning_rate": 9.779290132826224e-08,
      "logits/chosen": 1.1640625,
      "logits/rejected": 1.1875,
      "logps/chosen": -6.21875,
      "logps/rejected": -15.25,
      "loss": 1.1976,
      "nll_loss": 0.5625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.453125,
      "rewards/margins": 0.83984375,
      "rewards/rejected": 1.609375,
      "step": 201
    },
    {
      "epoch": 0.808,
      "grad_norm": 12.358923809267944,
      "learning_rate": 9.39708626145751e-08,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.203125,
      "logps/chosen": -6.15625,
      "logps/rejected": -15.9375,
      "loss": 1.3051,
      "nll_loss": 0.60546875,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 2.4375,
      "rewards/margins": 0.890625,
      "rewards/rejected": 1.5546875,
      "step": 202
    },
    {
      "epoch": 0.812,
      "grad_norm": 12.346183865053801,
      "learning_rate": 9.021724973728634e-08,
      "logits/chosen": 1.21875,
      "logits/rejected": 1.2421875,
      "logps/chosen": -5.21875,
      "logps/rejected": -22.375,
      "loss": 1.2555,
      "nll_loss": 0.498046875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.234375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": 0.63671875,
      "step": 203
    },
    {
      "epoch": 0.816,
      "grad_norm": 12.034662551259737,
      "learning_rate": 8.653269527193568e-08,
      "logits/chosen": 1.1953125,
      "logits/rejected": 1.1953125,
      "logps/chosen": -5.21875,
      "logps/rejected": -19.75,
      "loss": 1.2054,
      "nll_loss": 0.482421875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.734375,
      "rewards/margins": 1.7421875,
      "rewards/rejected": 0.99609375,
      "step": 204
    },
    {
      "epoch": 0.82,
      "grad_norm": 11.034715459854144,
      "learning_rate": 8.291782015603176e-08,
      "logits/chosen": 1.1484375,
      "logits/rejected": 1.171875,
      "logps/chosen": -5.25,
      "logps/rejected": -13.3125,
      "loss": 1.2401,
      "nll_loss": 0.52734375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.25,
      "rewards/margins": 0.73046875,
      "rewards/rejected": 1.5078125,
      "step": 205
    },
    {
      "epoch": 0.824,
      "grad_norm": 34.96908950850571,
      "learning_rate": 7.937323358440934e-08,
      "logits/chosen": 1.140625,
      "logits/rejected": 1.1875,
      "logps/chosen": -3.765625,
      "logps/rejected": -19.0,
      "loss": 1.1788,
      "nll_loss": 0.37109375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 2.59375,
      "rewards/margins": 0.99609375,
      "rewards/rejected": 1.5859375,
      "step": 206
    },
    {
      "epoch": 0.828,
      "grad_norm": 13.161837149601567,
      "learning_rate": 7.589953290656532e-08,
      "logits/chosen": 1.1171875,
      "logits/rejected": 1.1328125,
      "logps/chosen": -3.546875,
      "logps/rejected": -22.0,
      "loss": 1.2688,
      "nll_loss": 0.361328125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.234375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": 0.54296875,
      "step": 207
    },
    {
      "epoch": 0.832,
      "grad_norm": 11.065362037869805,
      "learning_rate": 7.249730352598998e-08,
      "logits/chosen": 1.1484375,
      "logits/rejected": 1.1875,
      "logps/chosen": -4.71875,
      "logps/rejected": -22.5,
      "loss": 1.2256,
      "nll_loss": 0.419921875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.0625,
      "rewards/margins": 1.5625,
      "rewards/rejected": 0.50390625,
      "step": 208
    },
    {
      "epoch": 0.836,
      "grad_norm": 15.587435364904376,
      "learning_rate": 6.916711880151304e-08,
      "logits/chosen": 1.1171875,
      "logits/rejected": 1.109375,
      "logps/chosen": -7.25,
      "logps/rejected": -20.125,
      "loss": 1.3117,
      "nll_loss": 0.70703125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.0625,
      "rewards/margins": 0.875,
      "rewards/rejected": 1.1875,
      "step": 209
    },
    {
      "epoch": 0.84,
      "grad_norm": 13.795300358361615,
      "learning_rate": 6.59095399506781e-08,
      "logits/chosen": 1.15625,
      "logits/rejected": 1.15625,
      "logps/chosen": -5.5,
      "logps/rejected": -18.875,
      "loss": 1.2623,
      "nll_loss": 0.52734375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.390625,
      "rewards/margins": 1.1640625,
      "rewards/rejected": 1.2265625,
      "step": 210
    },
    {
      "epoch": 0.844,
      "grad_norm": 14.092757534806013,
      "learning_rate": 6.2725115955164e-08,
      "logits/chosen": 1.171875,
      "logits/rejected": 1.1796875,
      "logps/chosen": -4.625,
      "logps/rejected": -25.125,
      "loss": 1.2361,
      "nll_loss": 0.443359375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.25,
      "rewards/margins": 1.6484375,
      "rewards/rejected": 0.60546875,
      "step": 211
    },
    {
      "epoch": 0.848,
      "grad_norm": 10.562244070406441,
      "learning_rate": 5.961438346826792e-08,
      "logits/chosen": 1.1328125,
      "logits/rejected": 1.1796875,
      "logps/chosen": -4.34375,
      "logps/rejected": -19.5,
      "loss": 1.171,
      "nll_loss": 0.419921875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.234375,
      "rewards/margins": 1.265625,
      "rewards/rejected": 0.96875,
      "step": 212
    },
    {
      "epoch": 0.852,
      "grad_norm": 11.070817604602208,
      "learning_rate": 5.6577866724466296e-08,
      "logits/chosen": 1.109375,
      "logits/rejected": 1.1328125,
      "logps/chosen": -6.125,
      "logps/rejected": -34.0,
      "loss": 1.1387,
      "nll_loss": 0.5625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.96875,
      "rewards/margins": 2.578125,
      "rewards/rejected": -0.60546875,
      "step": 213
    },
    {
      "epoch": 0.856,
      "grad_norm": 15.61141696529417,
      "learning_rate": 5.361607745106816e-08,
      "logits/chosen": 1.21875,
      "logits/rejected": 1.2109375,
      "logps/chosen": -3.703125,
      "logps/rejected": -13.0625,
      "loss": 1.223,
      "nll_loss": 0.408203125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.453125,
      "rewards/margins": 1.015625,
      "rewards/rejected": 1.4375,
      "step": 214
    },
    {
      "epoch": 0.86,
      "grad_norm": 12.525723751388632,
      "learning_rate": 5.072951478197723e-08,
      "logits/chosen": 1.1171875,
      "logits/rejected": 1.1328125,
      "logps/chosen": -4.90625,
      "logps/rejected": -21.5,
      "loss": 1.2809,
      "nll_loss": 0.498046875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.21875,
      "rewards/margins": 1.3203125,
      "rewards/rejected": 0.89453125,
      "step": 215
    },
    {
      "epoch": 0.864,
      "grad_norm": 12.527213509383836,
      "learning_rate": 4.79186651735749e-08,
      "logits/chosen": 1.1640625,
      "logits/rejected": 1.171875,
      "logps/chosen": -4.1875,
      "logps/rejected": -14.4375,
      "loss": 1.1196,
      "nll_loss": 0.427734375,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 2.140625,
      "rewards/margins": 0.54296875,
      "rewards/rejected": 1.6015625,
      "step": 216
    },
    {
      "epoch": 0.868,
      "grad_norm": 11.692116725453017,
      "learning_rate": 4.518400232274078e-08,
      "logits/chosen": 1.09375,
      "logits/rejected": 1.1171875,
      "logps/chosen": -5.21875,
      "logps/rejected": -20.0,
      "loss": 1.2534,
      "nll_loss": 0.470703125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 1.9375,
      "rewards/margins": 0.93359375,
      "rewards/rejected": 1.0,
      "step": 217
    },
    {
      "epoch": 0.872,
      "grad_norm": 11.469140530461623,
      "learning_rate": 4.252598708702343e-08,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.1796875,
      "logps/chosen": -4.6875,
      "logps/rejected": -15.5625,
      "loss": 1.1908,
      "nll_loss": 0.47265625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.359375,
      "rewards/margins": 1.296875,
      "rewards/rejected": 1.0625,
      "step": 218
    },
    {
      "epoch": 0.876,
      "grad_norm": 14.44996786417216,
      "learning_rate": 3.994506740697406e-08,
      "logits/chosen": 1.25,
      "logits/rejected": 1.2265625,
      "logps/chosen": -4.78125,
      "logps/rejected": -19.0,
      "loss": 1.1148,
      "nll_loss": 0.447265625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.296875,
      "rewards/margins": 1.3125,
      "rewards/rejected": 0.98046875,
      "step": 219
    },
    {
      "epoch": 0.88,
      "grad_norm": 11.827189877497567,
      "learning_rate": 3.744167823065814e-08,
      "logits/chosen": 1.0703125,
      "logits/rejected": 1.1171875,
      "logps/chosen": -4.40625,
      "logps/rejected": -24.875,
      "loss": 1.2424,
      "nll_loss": 0.388671875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.8125,
      "rewards/margins": 1.5390625,
      "rewards/rejected": 0.2734375,
      "step": 220
    },
    {
      "epoch": 0.884,
      "grad_norm": 12.018027049330724,
      "learning_rate": 3.501624144035559e-08,
      "logits/chosen": 1.15625,
      "logits/rejected": 1.203125,
      "logps/chosen": -5.21875,
      "logps/rejected": -17.0,
      "loss": 1.2421,
      "nll_loss": 0.46484375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.265625,
      "rewards/margins": 1.0625,
      "rewards/rejected": 1.2109375,
      "step": 221
    },
    {
      "epoch": 0.888,
      "grad_norm": 12.005772353362655,
      "learning_rate": 3.2669165781463395e-08,
      "logits/chosen": 1.171875,
      "logits/rejected": 1.1875,
      "logps/chosen": -5.03125,
      "logps/rejected": -19.875,
      "loss": 1.301,
      "nll_loss": 0.50390625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.09375,
      "rewards/margins": 0.6953125,
      "rewards/rejected": 1.3984375,
      "step": 222
    },
    {
      "epoch": 0.892,
      "grad_norm": 10.902340209779693,
      "learning_rate": 3.0400846793612544e-08,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.2265625,
      "logps/chosen": -4.71875,
      "logps/rejected": -15.4375,
      "loss": 1.1767,
      "nll_loss": 0.447265625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.390625,
      "rewards/margins": 0.5390625,
      "rewards/rejected": 1.8515625,
      "step": 223
    },
    {
      "epoch": 0.896,
      "grad_norm": 12.355828877171342,
      "learning_rate": 2.8211666744009045e-08,
      "logits/chosen": 1.109375,
      "logits/rejected": 1.09375,
      "logps/chosen": -4.25,
      "logps/rejected": -21.625,
      "loss": 1.1988,
      "nll_loss": 0.38671875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.1875,
      "rewards/margins": 1.1953125,
      "rewards/rejected": 1.0,
      "step": 224
    },
    {
      "epoch": 0.9,
      "grad_norm": 13.420061159806268,
      "learning_rate": 2.6101994563013098e-08,
      "logits/chosen": 1.140625,
      "logits/rejected": 1.1640625,
      "logps/chosen": -3.953125,
      "logps/rejected": -21.0,
      "loss": 1.2244,
      "nll_loss": 0.39453125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.21875,
      "rewards/margins": 1.375,
      "rewards/rejected": 0.84375,
      "step": 225
    },
    {
      "epoch": 0.904,
      "grad_norm": 11.000297680662745,
      "learning_rate": 2.4072185781965238e-08,
      "logits/chosen": 1.125,
      "logits/rejected": 1.125,
      "logps/chosen": -5.96875,
      "logps/rejected": -23.375,
      "loss": 1.2071,
      "nll_loss": 0.54296875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.3125,
      "rewards/margins": 1.5859375,
      "rewards/rejected": 0.71875,
      "step": 226
    },
    {
      "epoch": 0.908,
      "grad_norm": 14.17266394056175,
      "learning_rate": 2.212258247327059e-08,
      "logits/chosen": 1.125,
      "logits/rejected": 1.109375,
      "logps/chosen": -3.75,
      "logps/rejected": -22.5,
      "loss": 1.171,
      "nll_loss": 0.3828125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.15625,
      "rewards/margins": 1.6484375,
      "rewards/rejected": 0.51171875,
      "step": 227
    },
    {
      "epoch": 0.912,
      "grad_norm": 11.33899892588413,
      "learning_rate": 2.025351319275137e-08,
      "logits/chosen": 1.125,
      "logits/rejected": 1.1640625,
      "logps/chosen": -6.84375,
      "logps/rejected": -28.0,
      "loss": 1.2811,
      "nll_loss": 0.58203125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.171875,
      "rewards/margins": 2.03125,
      "rewards/rejected": 0.1337890625,
      "step": 228
    },
    {
      "epoch": 0.916,
      "grad_norm": 12.453357132102518,
      "learning_rate": 1.8465292924276844e-08,
      "logits/chosen": 1.0859375,
      "logits/rejected": 1.125,
      "logps/chosen": -5.9375,
      "logps/rejected": -32.75,
      "loss": 1.1113,
      "nll_loss": 0.5390625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.4375,
      "rewards/margins": 2.4375,
      "rewards/rejected": -3.0517578125e-05,
      "step": 229
    },
    {
      "epoch": 0.92,
      "grad_norm": 11.898178262181379,
      "learning_rate": 1.6758223026681507e-08,
      "logits/chosen": 1.1328125,
      "logits/rejected": 1.140625,
      "logps/chosen": -5.8125,
      "logps/rejected": -19.125,
      "loss": 1.2299,
      "nll_loss": 0.54296875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.8984375,
      "rewards/margins": 0.96875,
      "rewards/rejected": 0.9296875,
      "step": 230
    },
    {
      "epoch": 0.924,
      "grad_norm": 10.263964920299092,
      "learning_rate": 1.5132591182978105e-08,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.234375,
      "logps/chosen": -7.09375,
      "logps/rejected": -23.875,
      "loss": 1.2085,
      "nll_loss": 0.640625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.359375,
      "rewards/margins": 1.34375,
      "rewards/rejected": 1.0234375,
      "step": 231
    },
    {
      "epoch": 0.928,
      "grad_norm": 12.654190061391278,
      "learning_rate": 1.3588671351876357e-08,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.2265625,
      "logps/chosen": -4.4375,
      "logps/rejected": -15.8125,
      "loss": 1.2357,
      "nll_loss": 0.443359375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.265625,
      "rewards/margins": 0.76953125,
      "rewards/rejected": 1.4921875,
      "step": 232
    },
    {
      "epoch": 0.932,
      "grad_norm": 14.352106481713815,
      "learning_rate": 1.2126723721614052e-08,
      "logits/chosen": 1.15625,
      "logits/rejected": 1.1484375,
      "logps/chosen": -4.59375,
      "logps/rejected": -20.5,
      "loss": 1.2552,
      "nll_loss": 0.458984375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.0625,
      "rewards/margins": 1.1484375,
      "rewards/rejected": 0.921875,
      "step": 233
    },
    {
      "epoch": 0.936,
      "grad_norm": 11.269660437873329,
      "learning_rate": 1.0746994666109233e-08,
      "logits/chosen": 1.1875,
      "logits/rejected": 1.1875,
      "logps/chosen": -4.5625,
      "logps/rejected": -12.6875,
      "loss": 1.1757,
      "nll_loss": 0.416015625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.546875,
      "rewards/margins": 0.81640625,
      "rewards/rejected": 1.734375,
      "step": 234
    },
    {
      "epoch": 0.94,
      "grad_norm": 12.81841594760115,
      "learning_rate": 9.449716703439803e-09,
      "logits/chosen": 1.0546875,
      "logits/rejected": 1.046875,
      "logps/chosen": -5.09375,
      "logps/rejected": -14.875,
      "loss": 1.2255,
      "nll_loss": 0.466796875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.625,
      "rewards/margins": 1.2578125,
      "rewards/rejected": 1.3671875,
      "step": 235
    },
    {
      "epoch": 0.944,
      "grad_norm": 10.920504150516587,
      "learning_rate": 8.235108456658812e-09,
      "logits/chosen": 1.1640625,
      "logits/rejected": 1.15625,
      "logps/chosen": -4.59375,
      "logps/rejected": -27.75,
      "loss": 1.1622,
      "nll_loss": 0.4375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.078125,
      "rewards/margins": 1.859375,
      "rewards/rejected": 0.2138671875,
      "step": 236
    },
    {
      "epoch": 0.948,
      "grad_norm": 17.488700410279908,
      "learning_rate": 7.103374616951041e-09,
      "logits/chosen": 1.15625,
      "logits/rejected": 1.1484375,
      "logps/chosen": -4.40625,
      "logps/rejected": -13.5625,
      "loss": 1.2803,
      "nll_loss": 0.43359375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.546875,
      "rewards/margins": 0.8125,
      "rewards/rejected": 1.7421875,
      "step": 237
    },
    {
      "epoch": 0.952,
      "grad_norm": 15.482877470290028,
      "learning_rate": 6.054705909137425e-09,
      "logits/chosen": 1.1171875,
      "logits/rejected": 1.125,
      "logps/chosen": -4.75,
      "logps/rejected": -24.125,
      "loss": 1.2733,
      "nll_loss": 0.50390625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.125,
      "rewards/margins": 1.78125,
      "rewards/rejected": 0.349609375,
      "step": 238
    },
    {
      "epoch": 0.956,
      "grad_norm": 11.061399534591029,
      "learning_rate": 5.0892790595336575e-09,
      "logits/chosen": 1.109375,
      "logits/rejected": 1.1484375,
      "logps/chosen": -5.125,
      "logps/rejected": -19.375,
      "loss": 1.139,
      "nll_loss": 0.5078125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.109375,
      "rewards/margins": 1.4140625,
      "rewards/rejected": 0.6953125,
      "step": 239
    },
    {
      "epoch": 0.96,
      "grad_norm": 11.714703245702147,
      "learning_rate": 4.207256766166845e-09,
      "logits/chosen": 1.2109375,
      "logits/rejected": 1.1875,
      "logps/chosen": -5.53125,
      "logps/rejected": -18.375,
      "loss": 1.2334,
      "nll_loss": 0.51953125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.015625,
      "rewards/margins": 1.0546875,
      "rewards/rejected": 0.953125,
      "step": 240
    },
    {
      "epoch": 0.964,
      "grad_norm": 10.51394851397197,
      "learning_rate": 3.4087876713574935e-09,
      "logits/chosen": 1.1484375,
      "logits/rejected": 1.0859375,
      "logps/chosen": -5.8125,
      "logps/rejected": -21.5,
      "loss": 1.1431,
      "nll_loss": 0.59765625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.40625,
      "rewards/margins": 1.7265625,
      "rewards/rejected": 0.6796875,
      "step": 241
    },
    {
      "epoch": 0.968,
      "grad_norm": 13.245501985890717,
      "learning_rate": 2.69400633666933e-09,
      "logits/chosen": 1.1328125,
      "logits/rejected": 1.15625,
      "logps/chosen": -4.40625,
      "logps/rejected": -14.125,
      "loss": 1.2378,
      "nll_loss": 0.40625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.8984375,
      "rewards/margins": 0.6796875,
      "rewards/rejected": 1.21875,
      "step": 242
    },
    {
      "epoch": 0.972,
      "grad_norm": 11.496395504691947,
      "learning_rate": 2.0630332202324906e-09,
      "logits/chosen": 1.15625,
      "logits/rejected": 1.140625,
      "logps/chosen": -6.15625,
      "logps/rejected": -27.0,
      "loss": 1.2662,
      "nll_loss": 0.55078125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.28125,
      "rewards/margins": 1.8515625,
      "rewards/rejected": 0.42578125,
      "step": 243
    },
    {
      "epoch": 0.976,
      "grad_norm": 11.885785036184581,
      "learning_rate": 1.5159746564433196e-09,
      "logits/chosen": 1.1796875,
      "logits/rejected": 1.171875,
      "logps/chosen": -5.6875,
      "logps/rejected": -20.375,
      "loss": 1.2716,
      "nll_loss": 0.546875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.984375,
      "rewards/margins": 1.328125,
      "rewards/rejected": 0.6484375,
      "step": 244
    },
    {
      "epoch": 0.98,
      "grad_norm": 14.24854333069195,
      "learning_rate": 1.0529228380445899e-09,
      "logits/chosen": 1.140625,
      "logits/rejected": 1.1796875,
      "logps/chosen": -5.34375,
      "logps/rejected": -28.25,
      "loss": 1.1959,
      "nll_loss": 0.49609375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.328125,
      "rewards/margins": 2.015625,
      "rewards/rejected": 0.31640625,
      "step": 245
    },
    {
      "epoch": 0.984,
      "grad_norm": 12.20087123285817,
      "learning_rate": 6.739558005884883e-10,
      "logits/chosen": 1.1953125,
      "logits/rejected": 1.203125,
      "logps/chosen": -6.1875,
      "logps/rejected": -20.25,
      "loss": 1.2076,
      "nll_loss": 0.5546875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.1875,
      "rewards/margins": 1.140625,
      "rewards/rejected": 1.046875,
      "step": 246
    },
    {
      "epoch": 0.988,
      "grad_norm": 13.016658555990439,
      "learning_rate": 3.791374092859678e-10,
      "logits/chosen": 1.0546875,
      "logits/rejected": 1.0390625,
      "logps/chosen": -7.15625,
      "logps/rejected": -21.5,
      "loss": 1.2678,
      "nll_loss": 0.640625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 1.953125,
      "rewards/margins": 1.265625,
      "rewards/rejected": 0.6875,
      "step": 247
    },
    {
      "epoch": 0.992,
      "grad_norm": 13.049688068801364,
      "learning_rate": 1.6851734824380182e-10,
      "logits/chosen": 1.203125,
      "logits/rejected": 1.203125,
      "logps/chosen": -3.828125,
      "logps/rejected": -27.0,
      "loss": 1.3064,
      "nll_loss": 0.3671875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.421875,
      "rewards/margins": 1.765625,
      "rewards/rejected": 0.65625,
      "step": 248
    },
    {
      "epoch": 0.996,
      "grad_norm": 11.481752941680474,
      "learning_rate": 4.213111209155906e-11,
      "logits/chosen": 1.0859375,
      "logits/rejected": 1.1015625,
      "logps/chosen": -4.34375,
      "logps/rejected": -20.0,
      "loss": 1.153,
      "nll_loss": 0.3984375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.328125,
      "rewards/margins": 1.4765625,
      "rewards/rejected": 0.85546875,
      "step": 249
    },
    {
      "epoch": 1.0,
      "grad_norm": 12.415871966366533,
      "learning_rate": 0.0,
      "logits/chosen": 1.1484375,
      "logits/rejected": 1.140625,
      "logps/chosen": -4.96875,
      "logps/rejected": -21.75,
      "loss": 1.1798,
      "nll_loss": 0.482421875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.921875,
      "rewards/margins": 1.328125,
      "rewards/rejected": 0.59765625,
      "step": 250
    },
    {
      "epoch": 1.0,
      "step": 250,
      "total_flos": 1.540582387141627e+20,
      "train_loss": 1.5058151931762696,
      "train_runtime": 11811.4633,
      "train_samples_per_second": 5.418,
      "train_steps_per_second": 0.021
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 250,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100.0,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1.540582387141627e+20,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}