{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9765925925925925,
  "eval_steps": 500,
  "global_step": 315,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 1346.035400390625,
      "learning_rate": 7.8125e-06,
      "log_odds_chosen": -1.0068690776824951,
      "log_odds_ratio": -10.175436019897461,
      "logps/chosen": -21.69235610961914,
      "logps/rejected": -20.685726165771484,
      "loss": 709.8242,
      "nll_loss": 9.674795150756836,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -10.84617805480957,
      "rewards/margins": -0.5033147931098938,
      "rewards/rejected": -10.342863082885742,
      "step": 5
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 1455.1102294921875,
      "learning_rate": 1.5625e-05,
      "log_odds_chosen": -2.8008556365966797,
      "log_odds_ratio": -10.15751838684082,
      "logps/chosen": -20.97389793395996,
      "logps/rejected": -18.17340660095215,
      "loss": 714.7821,
      "nll_loss": 8.685011863708496,
      "rewards/accuracies": 0.44062501192092896,
      "rewards/chosen": -10.48694896697998,
      "rewards/margins": -1.4002447128295898,
      "rewards/rejected": -9.086703300476074,
      "step": 10
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 638.308837890625,
      "learning_rate": 2.34375e-05,
      "log_odds_chosen": -1.2725921869277954,
      "log_odds_ratio": -10.43528938293457,
      "logps/chosen": -20.847124099731445,
      "logps/rejected": -19.573030471801758,
      "loss": 687.155,
      "nll_loss": 8.07744312286377,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -10.423562049865723,
      "rewards/margins": -0.6370473504066467,
      "rewards/rejected": -9.786515235900879,
      "step": 15
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 5364.12744140625,
      "learning_rate": 3.125e-05,
      "log_odds_chosen": -3.7516868114471436,
      "log_odds_ratio": -8.449677467346191,
      "logps/chosen": -15.886372566223145,
      "logps/rejected": -12.137221336364746,
      "loss": 567.2934,
      "nll_loss": 6.516029357910156,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -7.943186283111572,
      "rewards/margins": -1.8745750188827515,
      "rewards/rejected": -6.068610668182373,
      "step": 20
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 732.103271484375,
      "learning_rate": 3.90625e-05,
      "log_odds_chosen": -0.05239830166101456,
      "log_odds_ratio": -1.5554496049880981,
      "logps/chosen": -3.6205036640167236,
      "logps/rejected": -3.5546963214874268,
      "loss": 117.5423,
      "nll_loss": 3.4973304271698,
      "rewards/accuracies": 0.5531250238418579,
      "rewards/chosen": -1.8102518320083618,
      "rewards/margins": -0.03290349990129471,
      "rewards/rejected": -1.7773481607437134,
      "step": 25
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 235.78294372558594,
      "learning_rate": 4.6875e-05,
      "log_odds_chosen": -0.004375058226287365,
      "log_odds_ratio": -0.9642303586006165,
      "logps/chosen": -2.1026854515075684,
      "logps/rejected": -2.0986106395721436,
      "loss": 67.608,
      "nll_loss": 2.820711851119995,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.0513427257537842,
      "rewards/margins": -0.002037324011325836,
      "rewards/rejected": -1.0493053197860718,
      "step": 30
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 201.821533203125,
      "learning_rate": 4.998613757348784e-05,
      "log_odds_chosen": 0.23235782980918884,
      "log_odds_ratio": -0.796781063079834,
      "logps/chosen": -1.6877104043960571,
      "logps/rejected": -1.896958589553833,
      "loss": 51.2266,
      "nll_loss": 2.3120052814483643,
      "rewards/accuracies": 0.5093749761581421,
      "rewards/chosen": -0.8438552021980286,
      "rewards/margins": 0.10462428629398346,
      "rewards/rejected": -0.9484792947769165,
      "step": 35
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 501.2691345214844,
      "learning_rate": 4.990147841143462e-05,
      "log_odds_chosen": 0.3352593779563904,
      "log_odds_ratio": -0.7122059464454651,
      "logps/chosen": -1.5017939805984497,
      "logps/rejected": -1.7836459875106812,
      "loss": 44.764,
      "nll_loss": 2.343039035797119,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7508969902992249,
      "rewards/margins": 0.1409260481595993,
      "rewards/rejected": -0.8918229937553406,
      "step": 40
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 224.84271240234375,
      "learning_rate": 4.97401218720448e-05,
      "log_odds_chosen": 0.3514587879180908,
      "log_odds_ratio": -0.6827880144119263,
      "logps/chosen": -1.4357855319976807,
      "logps/rejected": -1.7441604137420654,
      "loss": 42.3725,
      "nll_loss": 2.1640477180480957,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7178927659988403,
      "rewards/margins": 0.15418751537799835,
      "rewards/rejected": -0.8720802068710327,
      "step": 45
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 208.4950714111328,
      "learning_rate": 4.9502564938797946e-05,
      "log_odds_chosen": 0.3455984592437744,
      "log_odds_ratio": -0.6899778842926025,
      "logps/chosen": -1.336740493774414,
      "logps/rejected": -1.6336427927017212,
      "loss": 39.3189,
      "nll_loss": 2.020599842071533,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -0.668370246887207,
      "rewards/margins": 0.14845120906829834,
      "rewards/rejected": -0.8168213963508606,
      "step": 50
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 1214.6304931640625,
      "learning_rate": 4.918953929490768e-05,
      "log_odds_chosen": 1.7929474115371704,
      "log_odds_ratio": -0.681054413318634,
      "logps/chosen": -1.559952735900879,
      "logps/rejected": -3.2753818035125732,
      "loss": 24.7409,
      "nll_loss": 2.3052754402160645,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -0.7799763679504395,
      "rewards/margins": 0.8577146530151367,
      "rewards/rejected": -1.6376909017562866,
      "step": 55
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 1253.04541015625,
      "learning_rate": 4.88020090697132e-05,
      "log_odds_chosen": 6.016274452209473,
      "log_odds_ratio": -0.6842174530029297,
      "logps/chosen": -2.042940616607666,
      "logps/rejected": -7.971026420593262,
      "loss": -26.6321,
      "nll_loss": 2.7467756271362305,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.021470308303833,
      "rewards/margins": 2.964043140411377,
      "rewards/rejected": -3.985513210296631,
      "step": 60
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 22529.046875,
      "learning_rate": 4.834116786912897e-05,
      "log_odds_chosen": 13.827421188354492,
      "log_odds_ratio": -6.549749851226807,
      "logps/chosen": -11.553215026855469,
      "logps/rejected": -25.299327850341797,
      "loss": 152.311,
      "nll_loss": 9.787839889526367,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -5.776607513427734,
      "rewards/margins": 6.873056888580322,
      "rewards/rejected": -12.649663925170898,
      "step": 65
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 6729.6181640625,
      "learning_rate": 4.7808435099299045e-05,
      "log_odds_chosen": 9.947161674499512,
      "log_odds_ratio": -1.2594565153121948,
      "logps/chosen": -2.880189895629883,
      "logps/rejected": -12.729331970214844,
      "loss": -62.2908,
      "nll_loss": 3.341439723968506,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4400949478149414,
      "rewards/margins": 4.9245710372924805,
      "rewards/rejected": -6.364665985107422,
      "step": 70
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 9763.5361328125,
      "learning_rate": 4.720545159477922e-05,
      "log_odds_chosen": 9.978219985961914,
      "log_odds_ratio": -2.5337777137756348,
      "logps/chosen": -4.444361686706543,
      "logps/rejected": -14.362889289855957,
      "loss": -14.9101,
      "nll_loss": 4.244600296020508,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -2.2221808433532715,
      "rewards/margins": 4.959264278411865,
      "rewards/rejected": -7.1814446449279785,
      "step": 75
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 19205.619140625,
      "learning_rate": 4.653407456471222e-05,
      "log_odds_chosen": 9.231340408325195,
      "log_odds_ratio": -3.549084186553955,
      "logps/chosen": -9.340559959411621,
      "logps/rejected": -18.532567977905273,
      "loss": 153.5262,
      "nll_loss": 9.000253677368164,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -4.6702799797058105,
      "rewards/margins": 4.596003532409668,
      "rewards/rejected": -9.266283988952637,
      "step": 80
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 1391.7034912109375,
      "learning_rate": 4.579637187256222e-05,
      "log_odds_chosen": 6.83356237411499,
      "log_odds_ratio": -0.8187046051025391,
      "logps/chosen": -1.6173499822616577,
      "logps/rejected": -8.405004501342773,
      "loss": -55.3577,
      "nll_loss": 2.6606650352478027,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.8086749911308289,
      "rewards/margins": 3.393826961517334,
      "rewards/rejected": -4.202502250671387,
      "step": 85
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 714.6627197265625,
      "learning_rate": 4.499461566702685e-05,
      "log_odds_chosen": 12.91296100616455,
      "log_odds_ratio": -0.7300605177879333,
      "logps/chosen": -3.667490005493164,
      "logps/rejected": -16.505706787109375,
      "loss": -86.1328,
      "nll_loss": 4.034636974334717,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -1.833745002746582,
      "rewards/margins": 6.419107913970947,
      "rewards/rejected": -8.252853393554688,
      "step": 90
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 5335.16162109375,
      "learning_rate": 4.413127538374411e-05,
      "log_odds_chosen": 21.759021759033203,
      "log_odds_ratio": -5.81619930267334,
      "logps/chosen": -8.227999687194824,
      "logps/rejected": -29.910602569580078,
      "loss": -80.9532,
      "nll_loss": 7.397497653961182,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -4.113999843597412,
      "rewards/margins": 10.841300010681152,
      "rewards/rejected": -14.955301284790039,
      "step": 95
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 9169.87890625,
      "learning_rate": 4.320901013934887e-05,
      "log_odds_chosen": 21.403528213500977,
      "log_odds_ratio": -4.2520060539245605,
      "logps/chosen": -6.323123931884766,
      "logps/rejected": -27.63702964782715,
      "loss": -135.6809,
      "nll_loss": 5.874188423156738,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": -3.161561965942383,
      "rewards/margins": 10.656951904296875,
      "rewards/rejected": -13.818514823913574,
      "step": 100
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 15093.3359375,
      "learning_rate": 4.223066054130568e-05,
      "log_odds_chosen": 21.14289093017578,
      "log_odds_ratio": -2.737898349761963,
      "logps/chosen": -6.241971015930176,
      "logps/rejected": -27.327472686767578,
      "loss": -135.2989,
      "nll_loss": 6.4709601402282715,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -3.120985507965088,
      "rewards/margins": 10.542750358581543,
      "rewards/rejected": -13.663736343383789,
      "step": 105
    },
    {
      "epoch": 1.037925925925926,
      "grad_norm": 2860.319580078125,
      "learning_rate": 4.1199239938743797e-05,
      "log_odds_chosen": 14.374427795410156,
      "log_odds_ratio": -0.5818507075309753,
      "logps/chosen": -2.059671401977539,
      "logps/rejected": -16.356985092163086,
      "loss": -143.7255,
      "nll_loss": 2.7815299034118652,
      "rewards/accuracies": 0.6538461446762085,
      "rewards/chosen": -1.0298357009887695,
      "rewards/margins": 7.148656368255615,
      "rewards/rejected": -8.178492546081543,
      "step": 110
    },
    {
      "epoch": 1.0853333333333333,
      "grad_norm": 32810.50390625,
      "learning_rate": 4.0117925141242174e-05,
      "log_odds_chosen": 27.462366104125977,
      "log_odds_ratio": -3.54345703125,
      "logps/chosen": -7.290637016296387,
      "logps/rejected": -34.6160888671875,
      "loss": -200.7838,
      "nll_loss": 7.086569786071777,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.6453185081481934,
      "rewards/margins": 13.662725448608398,
      "rewards/rejected": -17.30804443359375,
      "step": 115
    },
    {
      "epoch": 1.1327407407407408,
      "grad_norm": 2398.583984375,
      "learning_rate": 3.899004663415084e-05,
      "log_odds_chosen": 17.45090675354004,
      "log_odds_ratio": -4.304950714111328,
      "logps/chosen": -7.110805511474609,
      "logps/rejected": -24.452959060668945,
      "loss": -47.5812,
      "nll_loss": 7.3393707275390625,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -3.5554027557373047,
      "rewards/margins": 8.671076774597168,
      "rewards/rejected": -12.226479530334473,
      "step": 120
    },
    {
      "epoch": 1.1801481481481482,
      "grad_norm": 23893.76171875,
      "learning_rate": 3.781907832058587e-05,
      "log_odds_chosen": 24.439823150634766,
      "log_odds_ratio": -5.230821132659912,
      "logps/chosen": -8.570024490356445,
      "logps/rejected": -32.90158462524414,
      "loss": -112.3321,
      "nll_loss": 8.563321113586426,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.285012245178223,
      "rewards/margins": 12.165781021118164,
      "rewards/rejected": -16.45079231262207,
      "step": 125
    },
    {
      "epoch": 1.2275555555555555,
      "grad_norm": 2984.347900390625,
      "learning_rate": 3.660862682169282e-05,
      "log_odds_chosen": 31.73781967163086,
      "log_odds_ratio": -3.9809155464172363,
      "logps/chosen": -10.971330642700195,
      "logps/rejected": -42.56734085083008,
      "loss": -150.9388,
      "nll_loss": 10.558366775512695,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -5.485665321350098,
      "rewards/margins": 15.798006057739258,
      "rewards/rejected": -21.28367042541504,
      "step": 130
    },
    {
      "epoch": 1.274962962962963,
      "grad_norm": 8554.173828125,
      "learning_rate": 3.5362420368134356e-05,
      "log_odds_chosen": 28.130645751953125,
      "log_odds_ratio": -1.710423231124878,
      "logps/chosen": -3.9046826362609863,
      "logps/rejected": -31.89151382446289,
      "loss": -319.8829,
      "nll_loss": 3.7381324768066406,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -1.9523413181304932,
      "rewards/margins": 13.993415832519531,
      "rewards/rejected": -15.945756912231445,
      "step": 135
    },
    {
      "epoch": 1.3223703703703704,
      "grad_norm": 4571.39404296875,
      "learning_rate": 3.408429731701635e-05,
      "log_odds_chosen": 23.899776458740234,
      "log_odds_ratio": -3.7175846099853516,
      "logps/chosen": -5.324652671813965,
      "logps/rejected": -29.102670669555664,
      "loss": -207.4437,
      "nll_loss": 5.260149955749512,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -2.6623263359069824,
      "rewards/margins": 11.889008522033691,
      "rewards/rejected": -14.551335334777832,
      "step": 140
    },
    {
      "epoch": 1.3697777777777778,
      "grad_norm": 2103.40673828125,
      "learning_rate": 3.2778194329621104e-05,
      "log_odds_chosen": 20.076154708862305,
      "log_odds_ratio": -2.8479440212249756,
      "logps/chosen": -3.987905979156494,
      "logps/rejected": -23.93975257873535,
      "loss": -188.7188,
      "nll_loss": 6.649003505706787,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": -1.993952989578247,
      "rewards/margins": 9.975922584533691,
      "rewards/rejected": -11.969876289367676,
      "step": 145
    },
    {
      "epoch": 1.417185185185185,
      "grad_norm": 21103.2109375,
      "learning_rate": 3.144813424636031e-05,
      "log_odds_chosen": 26.536422729492188,
      "log_odds_ratio": -3.6103949546813965,
      "logps/chosen": -7.1077470779418945,
      "logps/rejected": -33.501373291015625,
      "loss": -191.7946,
      "nll_loss": 10.428869247436523,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -3.5538735389709473,
      "rewards/margins": 13.196813583374023,
      "rewards/rejected": -16.750686645507812,
      "step": 150
    },
    {
      "epoch": 1.4645925925925927,
      "grad_norm": 20648.404296875,
      "learning_rate": 3.0098213696293542e-05,
      "log_odds_chosen": 30.618228912353516,
      "log_odds_ratio": -8.441522598266602,
      "logps/chosen": -22.582386016845703,
      "logps/rejected": -53.13446044921875,
      "loss": 236.5034,
      "nll_loss": 25.505130767822266,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -11.291193008422852,
      "rewards/margins": 15.276036262512207,
      "rewards/rejected": -26.567230224609375,
      "step": 155
    },
    {
      "epoch": 1.512,
      "grad_norm": 6142.369140625,
      "learning_rate": 2.8732590479375165e-05,
      "log_odds_chosen": 28.4414005279541,
      "log_odds_ratio": -4.836407661437988,
      "logps/chosen": -10.202522277832031,
      "logps/rejected": -38.521305084228516,
      "loss": -124.1278,
      "nll_loss": 7.826313018798828,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -5.101261138916016,
      "rewards/margins": 14.159391403198242,
      "rewards/rejected": -19.260652542114258,
      "step": 160
    },
    {
      "epoch": 1.5594074074074074,
      "grad_norm": 887.5225219726562,
      "learning_rate": 2.7355470760292956e-05,
      "log_odds_chosen": 15.544232368469238,
      "log_odds_ratio": -2.6855194568634033,
      "logps/chosen": -3.4005138874053955,
      "logps/rejected": -18.811342239379883,
      "loss": -135.4756,
      "nll_loss": 4.0409064292907715,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -1.7002569437026978,
      "rewards/margins": 7.705413818359375,
      "rewards/rejected": -9.405671119689941,
      "step": 165
    },
    {
      "epoch": 1.6068148148148147,
      "grad_norm": 1129.25048828125,
      "learning_rate": 2.597109611334169e-05,
      "log_odds_chosen": 17.921756744384766,
      "log_odds_ratio": -2.1972568035125732,
      "logps/chosen": -3.035590171813965,
      "logps/rejected": -20.839210510253906,
      "loss": -185.0614,
      "nll_loss": 3.198902130126953,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5177950859069824,
      "rewards/margins": 8.901809692382812,
      "rewards/rejected": -10.419605255126953,
      "step": 170
    },
    {
      "epoch": 1.6542222222222223,
      "grad_norm": 6776.86669921875,
      "learning_rate": 2.458373045823404e-05,
      "log_odds_chosen": 19.142057418823242,
      "log_odds_ratio": -4.326524257659912,
      "logps/chosen": -8.229945182800293,
      "logps/rejected": -27.229846954345703,
      "loss": -37.5736,
      "nll_loss": 9.928422927856445,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -4.1149725914001465,
      "rewards/margins": 9.499950408935547,
      "rewards/rejected": -13.614923477172852,
      "step": 175
    },
    {
      "epoch": 1.7016296296296296,
      "grad_norm": 6735.8046875,
      "learning_rate": 2.3197646927086697e-05,
      "log_odds_chosen": 21.28363609313965,
      "log_odds_ratio": -3.7137675285339355,
      "logps/chosen": -6.68829345703125,
      "logps/rejected": -27.832469940185547,
      "loss": -121.5285,
      "nll_loss": 10.662646293640137,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.344146728515625,
      "rewards/margins": 10.572088241577148,
      "rewards/rejected": -13.916234970092773,
      "step": 180
    },
    {
      "epoch": 1.749037037037037,
      "grad_norm": 980.9172973632812,
      "learning_rate": 2.1817114703032176e-05,
      "log_odds_chosen": 11.027368545532227,
      "log_odds_ratio": -1.5065878629684448,
      "logps/chosen": -2.5348973274230957,
      "logps/rejected": -13.437909126281738,
      "loss": -90.6843,
      "nll_loss": 4.682984828948975,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.2674486637115479,
      "rewards/margins": 5.4515061378479,
      "rewards/rejected": -6.718954563140869,
      "step": 185
    },
    {
      "epoch": 1.7964444444444445,
      "grad_norm": 2794.48828125,
      "learning_rate": 2.0446385870993467e-05,
      "log_odds_chosen": 25.16632080078125,
      "log_odds_ratio": -1.736579179763794,
      "logps/chosen": -2.6034204959869385,
      "logps/rejected": -27.604034423828125,
      "loss": -313.2717,
      "nll_loss": 4.227630138397217,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.3017102479934692,
      "rewards/margins": 12.5003080368042,
      "rewards/rejected": -13.802017211914062,
      "step": 190
    },
    {
      "epoch": 1.8438518518518519,
      "grad_norm": 18748.30859375,
      "learning_rate": 1.9089682321121834e-05,
      "log_odds_chosen": 25.935617446899414,
      "log_odds_ratio": -5.700267791748047,
      "logps/chosen": -7.655577182769775,
      "logps/rejected": -33.431495666503906,
      "loss": -164.2176,
      "nll_loss": 11.73558235168457,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -3.8277885913848877,
      "rewards/margins": 12.887959480285645,
      "rewards/rejected": -16.715747833251953,
      "step": 195
    },
    {
      "epoch": 1.8912592592592592,
      "grad_norm": 9071.26953125,
      "learning_rate": 1.775118274523545e-05,
      "log_odds_chosen": 29.912353515625,
      "log_odds_ratio": -3.8106212615966797,
      "logps/chosen": -7.143023490905762,
      "logps/rejected": -36.88776397705078,
      "loss": -243.6502,
      "nll_loss": 12.543132781982422,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -3.571511745452881,
      "rewards/margins": 14.872370719909668,
      "rewards/rejected": -18.44388198852539,
      "step": 200
    },
    {
      "epoch": 1.9386666666666668,
      "grad_norm": 5131.81005859375,
      "learning_rate": 1.643500976631037e-05,
      "log_odds_chosen": 21.7247314453125,
      "log_odds_ratio": -4.9020280838012695,
      "logps/chosen": -8.960885047912598,
      "logps/rejected": -30.520559310913086,
      "loss": -54.6864,
      "nll_loss": 11.589077949523926,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -4.480442523956299,
      "rewards/margins": 10.779837608337402,
      "rewards/rejected": -15.260279655456543,
      "step": 205
    },
    {
      "epoch": 1.986074074074074,
      "grad_norm": 6081.4013671875,
      "learning_rate": 1.514521724066537e-05,
      "log_odds_chosen": 26.899578094482422,
      "log_odds_ratio": -4.448848724365234,
      "logps/chosen": -6.3961944580078125,
      "logps/rejected": -33.168235778808594,
      "loss": -220.8887,
      "nll_loss": 8.339305877685547,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -3.1980972290039062,
      "rewards/margins": 13.386022567749023,
      "rewards/rejected": -16.584117889404297,
      "step": 210
    },
    {
      "epoch": 2.0284444444444443,
      "grad_norm": 13102.798828125,
      "learning_rate": 1.3885777771950348e-05,
      "log_odds_chosen": 31.693286895751953,
      "log_odds_ratio": -7.093617916107178,
      "logps/chosen": -9.377802848815918,
      "logps/rejected": -40.8663330078125,
      "loss": -178.6939,
      "nll_loss": 11.103485107421875,
      "rewards/accuracies": 0.7062937021255493,
      "rewards/chosen": -4.688901424407959,
      "rewards/margins": 15.744266510009766,
      "rewards/rejected": -20.43316650390625,
      "step": 215
    },
    {
      "epoch": 2.075851851851852,
      "grad_norm": 16916.310546875,
      "learning_rate": 1.2660570475395683e-05,
      "log_odds_chosen": 29.3436222076416,
      "log_odds_ratio": -5.789451599121094,
      "logps/chosen": -14.939682006835938,
      "logps/rejected": -44.14417266845703,
      "loss": 14.2524,
      "nll_loss": 22.04985809326172,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -7.469841003417969,
      "rewards/margins": 14.60224437713623,
      "rewards/rejected": -22.072086334228516,
      "step": 220
    },
    {
      "epoch": 2.1232592592592594,
      "grad_norm": 17463.740234375,
      "learning_rate": 1.1473369030008974e-05,
      "log_odds_chosen": 36.83652877807617,
      "log_odds_ratio": -3.7113094329833984,
      "logps/chosen": -9.069849967956543,
      "logps/rejected": -45.71887969970703,
      "loss": -291.8311,
      "nll_loss": 16.01699447631836,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -4.5349249839782715,
      "rewards/margins": 18.324514389038086,
      "rewards/rejected": -22.859439849853516,
      "step": 225
    },
    {
      "epoch": 2.1706666666666665,
      "grad_norm": 2066.959228515625,
      "learning_rate": 1.0327830055518842e-05,
      "log_odds_chosen": 34.293540954589844,
      "log_odds_ratio": -2.4719889163970947,
      "logps/chosen": -6.306262016296387,
      "logps/rejected": -40.41852951049805,
      "loss": -340.089,
      "nll_loss": 12.120941162109375,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -3.1531310081481934,
      "rewards/margins": 17.056133270263672,
      "rewards/rejected": -20.209264755249023,
      "step": 230
    },
    {
      "epoch": 2.218074074074074,
      "grad_norm": 3184.418212890625,
      "learning_rate": 9.227481849865235e-06,
      "log_odds_chosen": 36.27004623413086,
      "log_odds_ratio": -2.707547903060913,
      "logps/chosen": -3.9607880115509033,
      "logps/rejected": -40.02045440673828,
      "loss": -446.1736,
      "nll_loss": 6.836036682128906,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -1.9803940057754517,
      "rewards/margins": 18.02983283996582,
      "rewards/rejected": -20.01022720336914,
      "step": 235
    },
    {
      "epoch": 2.2654814814814817,
      "grad_norm": 12459.7412109375,
      "learning_rate": 8.175713521924978e-06,
      "log_odds_chosen": 31.65945816040039,
      "log_odds_ratio": -2.4659223556518555,
      "logps/chosen": -4.006608963012695,
      "logps/rejected": -35.472171783447266,
      "loss": -371.1167,
      "nll_loss": 7.588781833648682,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -2.0033044815063477,
      "rewards/margins": 15.732783317565918,
      "rewards/rejected": -17.736085891723633,
      "step": 240
    },
    {
      "epoch": 2.3128888888888888,
      "grad_norm": 10665.607421875,
      "learning_rate": 7.1757645529443665e-06,
      "log_odds_chosen": 41.71519088745117,
      "log_odds_ratio": -2.4143545627593994,
      "logps/chosen": -5.24094295501709,
      "logps/rejected": -46.76899337768555,
      "loss": -492.2525,
      "nll_loss": 14.035673141479492,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -2.620471477508545,
      "rewards/margins": 20.764026641845703,
      "rewards/rejected": -23.384496688842773,
      "step": 245
    },
    {
      "epoch": 2.3602962962962963,
      "grad_norm": 15119.1689453125,
      "learning_rate": 6.230714818829733e-06,
      "log_odds_chosen": 35.395755767822266,
      "log_odds_ratio": -1.310185194015503,
      "logps/chosen": -5.026707649230957,
      "logps/rejected": -40.237152099609375,
      "loss": -398.4109,
      "nll_loss": 9.918493270874023,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -2.5133538246154785,
      "rewards/margins": 17.605222702026367,
      "rewards/rejected": -20.118576049804688,
      "step": 250
    },
    {
      "epoch": 2.407703703703704,
      "grad_norm": 5417.455078125,
      "learning_rate": 5.343475104027743e-06,
      "log_odds_chosen": 34.123165130615234,
      "log_odds_ratio": -2.2660939693450928,
      "logps/chosen": -6.2752251625061035,
      "logps/rejected": -40.19994354248047,
      "loss": -337.6817,
      "nll_loss": 11.942907333374023,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.1376125812530518,
      "rewards/margins": 16.962358474731445,
      "rewards/rejected": -20.099971771240234,
      "step": 255
    },
    {
      "epoch": 2.455111111111111,
      "grad_norm": 5251.4404296875,
      "learning_rate": 4.516778136213037e-06,
      "log_odds_chosen": 29.794910430908203,
      "log_odds_ratio": -3.3600330352783203,
      "logps/chosen": -5.655725955963135,
      "logps/rejected": -35.23265838623047,
      "loss": -288.0842,
      "nll_loss": 10.556007385253906,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.8278629779815674,
      "rewards/margins": 14.78846549987793,
      "rewards/rejected": -17.616329193115234,
      "step": 260
    },
    {
      "epoch": 2.5025185185185186,
      "grad_norm": 17512.310546875,
      "learning_rate": 3.7531701693965554e-06,
      "log_odds_chosen": 43.79105758666992,
      "log_odds_ratio": -4.43022346496582,
      "logps/chosen": -8.150279998779297,
      "logps/rejected": -51.75994110107422,
      "loss": -432.5288,
      "nll_loss": 18.760528564453125,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -4.075139999389648,
      "rewards/margins": 21.804828643798828,
      "rewards/rejected": -25.87997055053711,
      "step": 265
    },
    {
      "epoch": 2.549925925925926,
      "grad_norm": 16464.80859375,
      "learning_rate": 3.055003141378948e-06,
      "log_odds_chosen": 45.24549102783203,
      "log_odds_ratio": -4.265998840332031,
      "logps/chosen": -9.220375061035156,
      "logps/rejected": -54.259765625,
      "loss": -420.995,
      "nll_loss": 17.882661819458008,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -4.610187530517578,
      "rewards/margins": 22.51969337463379,
      "rewards/rejected": -27.1298828125,
      "step": 270
    },
    {
      "epoch": 2.5973333333333333,
      "grad_norm": 10890.8203125,
      "learning_rate": 2.424427429704365e-06,
      "log_odds_chosen": 35.03772735595703,
      "log_odds_ratio": -4.138024806976318,
      "logps/chosen": -8.157249450683594,
      "logps/rejected": -42.98029327392578,
      "loss": -291.3517,
      "nll_loss": 13.331751823425293,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -4.078624725341797,
      "rewards/margins": 17.411518096923828,
      "rewards/rejected": -21.49014663696289,
      "step": 275
    },
    {
      "epoch": 2.644740740740741,
      "grad_norm": 12637.064453125,
      "learning_rate": 1.8633852284264508e-06,
      "log_odds_chosen": 41.4226188659668,
      "log_odds_ratio": -3.3395819664001465,
      "logps/chosen": -5.7105512619018555,
      "logps/rejected": -46.93029022216797,
      "loss": -472.0542,
      "nll_loss": 11.638737678527832,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -2.8552756309509277,
      "rewards/margins": 20.609867095947266,
      "rewards/rejected": -23.465145111083984,
      "step": 280
    },
    {
      "epoch": 2.6921481481481484,
      "grad_norm": 18065.2109375,
      "learning_rate": 1.3736045660864034e-06,
      "log_odds_chosen": 33.556556701660156,
      "log_odds_ratio": -4.451475620269775,
      "logps/chosen": -7.034272193908691,
      "logps/rejected": -40.41550064086914,
      "loss": -304.8266,
      "nll_loss": 12.904256820678711,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.5171360969543457,
      "rewards/margins": 16.690614700317383,
      "rewards/rejected": -20.20775032043457,
      "step": 285
    },
    {
      "epoch": 2.7395555555555555,
      "grad_norm": 13026.53515625,
      "learning_rate": 9.565939833279192e-07,
      "log_odds_chosen": 35.401798248291016,
      "log_odds_ratio": -3.9767117500305176,
      "logps/chosen": -7.844551086425781,
      "logps/rejected": -43.053428649902344,
      "loss": -308.0105,
      "nll_loss": 11.636465072631836,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -3.9222755432128906,
      "rewards/margins": 17.604434967041016,
      "rewards/rejected": -21.526714324951172,
      "step": 290
    },
    {
      "epoch": 2.786962962962963,
      "grad_norm": 7266.41943359375,
      "learning_rate": 6.136378865420872e-07,
      "log_odds_chosen": 32.378849029541016,
      "log_odds_ratio": -2.3196144104003906,
      "logps/chosen": -5.068166255950928,
      "logps/rejected": -37.264400482177734,
      "loss": -348.6297,
      "nll_loss": 9.656694412231445,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -2.534083127975464,
      "rewards/margins": 16.098114013671875,
      "rewards/rejected": -18.632200241088867,
      "step": 295
    },
    {
      "epoch": 2.83437037037037,
      "grad_norm": 6610.1474609375,
      "learning_rate": 3.45792591853214e-07,
      "log_odds_chosen": 41.97068786621094,
      "log_odds_ratio": -1.2807286977767944,
      "logps/chosen": -3.7067933082580566,
      "logps/rejected": -45.47202682495117,
      "loss": -545.1357,
      "nll_loss": 8.846769332885742,
      "rewards/accuracies": 0.7406250238418579,
      "rewards/chosen": -1.8533966541290283,
      "rewards/margins": 20.88261604309082,
      "rewards/rejected": -22.736013412475586,
      "step": 300
    },
    {
      "epoch": 2.8817777777777778,
      "grad_norm": 8774.75390625,
      "learning_rate": 1.538830716302092e-07,
      "log_odds_chosen": 37.96703338623047,
      "log_odds_ratio": -2.458489179611206,
      "logps/chosen": -4.410444736480713,
      "logps/rejected": -42.13166046142578,
      "loss": -457.7542,
      "nll_loss": 8.809645652770996,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -2.2052223682403564,
      "rewards/margins": 18.860610961914062,
      "rewards/rejected": -21.06583023071289,
      "step": 305
    },
    {
      "epoch": 2.9291851851851853,
      "grad_norm": 5141.3994140625,
      "learning_rate": 3.8500413544415025e-08,
      "log_odds_chosen": 43.835670471191406,
      "log_odds_ratio": -3.51432466506958,
      "logps/chosen": -6.511708736419678,
      "logps/rejected": -50.12602996826172,
      "loss": -484.2083,
      "nll_loss": 11.03735637664795,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.255854368209839,
      "rewards/margins": 21.807161331176758,
      "rewards/rejected": -25.06301498413086,
      "step": 310
    },
    {
      "epoch": 2.9765925925925925,
      "grad_norm": 12163.37890625,
      "learning_rate": 0.0,
      "log_odds_chosen": 37.13484573364258,
      "log_odds_ratio": -1.1676459312438965,
      "logps/chosen": -3.6658377647399902,
      "logps/rejected": -40.594600677490234,
      "loss": -469.246,
      "nll_loss": 8.678117752075195,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8329188823699951,
      "rewards/margins": 18.46438217163086,
      "rewards/rejected": -20.297300338745117,
      "step": 315
    },
    {
      "epoch": 2.9765925925925925,
      "step": 315,
      "total_flos": 0.0,
      "train_loss": -125.51550612676711,
      "train_runtime": 9350.753,
      "train_samples_per_second": 2.166,
      "train_steps_per_second": 0.034
    }
  ],
  "logging_steps": 5,
  "max_steps": 315,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}