{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9765925925925925,
  "eval_steps": 500,
  "global_step": 315,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 1589.2005615234375,
      "learning_rate": 7.8125e-06,
      "log_odds_chosen": -1.069156289100647,
      "log_odds_ratio": -11.014480590820312,
      "logps/chosen": -22.73740005493164,
      "logps/rejected": -21.668460845947266,
      "loss": 713.2996,
      "nll_loss": 7.888848781585693,
      "po_loss": 14.401758193969727,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -11.36870002746582,
      "rewards/margins": -0.534469485282898,
      "rewards/rejected": -10.834230422973633,
      "step": 5
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 1627.2967529296875,
      "learning_rate": 1.5625e-05,
      "log_odds_chosen": -3.1202898025512695,
      "log_odds_ratio": -11.058849334716797,
      "logps/chosen": -22.057109832763672,
      "logps/rejected": -18.937150955200195,
      "loss": 699.5439,
      "nll_loss": 7.041332244873047,
      "po_loss": 14.819416999816895,
      "rewards/accuracies": 0.44062501192092896,
      "rewards/chosen": -11.028554916381836,
      "rewards/margins": -1.559978723526001,
      "rewards/rejected": -9.468575477600098,
      "step": 10
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 1247.8370361328125,
      "learning_rate": 2.34375e-05,
      "log_odds_chosen": -1.2872090339660645,
      "log_odds_ratio": -11.414179801940918,
      "logps/chosen": -22.184202194213867,
      "logps/rejected": -20.895626068115234,
      "loss": 729.5998,
      "nll_loss": 6.488010406494141,
      "po_loss": 16.311986923217773,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -11.092101097106934,
      "rewards/margins": -0.6442887187004089,
      "rewards/rejected": -10.447813034057617,
      "step": 15
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 4529.8486328125,
      "learning_rate": 3.125e-05,
      "log_odds_chosen": -4.82450008392334,
      "log_odds_ratio": -10.463640213012695,
      "logps/chosen": -18.870685577392578,
      "logps/rejected": -14.047442436218262,
      "loss": 641.8112,
      "nll_loss": 5.417292594909668,
      "po_loss": 14.639307022094727,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -9.435342788696289,
      "rewards/margins": -2.411621332168579,
      "rewards/rejected": -7.023721218109131,
      "step": 20
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 443.5966491699219,
      "learning_rate": 3.90625e-05,
      "log_odds_chosen": -0.13172200322151184,
      "log_odds_ratio": -2.3210015296936035,
      "logps/chosen": -5.285498142242432,
      "logps/rejected": -5.136414527893066,
      "loss": 166.894,
      "nll_loss": 2.6062519550323486,
      "po_loss": 2.6091866493225098,
      "rewards/accuracies": 0.5531250238418579,
      "rewards/chosen": -2.642749071121216,
      "rewards/margins": -0.07454162836074829,
      "rewards/rejected": -2.568207263946533,
      "step": 25
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 174.74203491210938,
      "learning_rate": 4.6875e-05,
      "log_odds_chosen": 0.12811218202114105,
      "log_odds_ratio": -0.844332218170166,
      "logps/chosen": -1.924329161643982,
      "logps/rejected": -2.0408449172973633,
      "loss": 58.317,
      "nll_loss": 1.7607396841049194,
      "po_loss": 0.06166689470410347,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.962164580821991,
      "rewards/margins": 0.05825779587030411,
      "rewards/rejected": -1.0204224586486816,
      "step": 30
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 361.8482666015625,
      "learning_rate": 4.998613757348784e-05,
      "log_odds_chosen": 0.24519693851470947,
      "log_odds_ratio": -0.7969210147857666,
      "logps/chosen": -1.7141778469085693,
      "logps/rejected": -1.9342174530029297,
      "loss": 51.3055,
      "nll_loss": 1.5798017978668213,
      "po_loss": 0.023495521396398544,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.8570889234542847,
      "rewards/margins": 0.11001990735530853,
      "rewards/rejected": -0.9671087265014648,
      "step": 35
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 124.48103332519531,
      "learning_rate": 4.990147841143462e-05,
      "log_odds_chosen": 0.2787536680698395,
      "log_odds_ratio": -0.7292624711990356,
      "logps/chosen": -1.5313652753829956,
      "logps/rejected": -1.765584945678711,
      "loss": 46.8311,
      "nll_loss": 1.4609569311141968,
      "po_loss": 0.0025150892324745655,
      "rewards/accuracies": 0.565625011920929,
      "rewards/chosen": -0.7656826376914978,
      "rewards/margins": 0.11710979789495468,
      "rewards/rejected": -0.8827924728393555,
      "step": 40
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 78.25408172607422,
      "learning_rate": 4.97401218720448e-05,
      "log_odds_chosen": 0.2244810163974762,
      "log_odds_ratio": -0.7177098393440247,
      "logps/chosen": -1.3968251943588257,
      "logps/rejected": -1.5895179510116577,
      "loss": 43.8314,
      "nll_loss": 1.3644416332244873,
      "po_loss": 0.005288129206746817,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.6984125971794128,
      "rewards/margins": 0.09634630382061005,
      "rewards/rejected": -0.7947589755058289,
      "step": 45
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 91.44141387939453,
      "learning_rate": 4.9502564938797946e-05,
      "log_odds_chosen": 0.24271154403686523,
      "log_odds_ratio": -0.695084810256958,
      "logps/chosen": -1.2792396545410156,
      "logps/rejected": -1.4807188510894775,
      "loss": 40.672,
      "nll_loss": 1.268882393836975,
      "po_loss": 0.0021188005339354277,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6396198272705078,
      "rewards/margins": 0.10073961317539215,
      "rewards/rejected": -0.7403594255447388,
      "step": 50
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 68.53915405273438,
      "learning_rate": 4.918953929490768e-05,
      "log_odds_chosen": 0.23904606699943542,
      "log_odds_ratio": -0.7103737592697144,
      "logps/chosen": -1.2774039506912231,
      "logps/rejected": -1.4675384759902954,
      "loss": 40.8361,
      "nll_loss": 1.2758907079696655,
      "po_loss": 0.0002387870626989752,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6387019753456116,
      "rewards/margins": 0.09506731480360031,
      "rewards/rejected": -0.7337692379951477,
      "step": 55
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 98.16889190673828,
      "learning_rate": 4.88020090697132e-05,
      "log_odds_chosen": 0.2052260935306549,
      "log_odds_ratio": -0.7101542949676514,
      "logps/chosen": -1.283363699913025,
      "logps/rejected": -1.4537694454193115,
      "loss": 41.0667,
      "nll_loss": 1.2872891426086426,
      "po_loss": -0.003953990060836077,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.6416818499565125,
      "rewards/margins": 0.08520293235778809,
      "rewards/rejected": -0.7268847227096558,
      "step": 60
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 94.77367401123047,
      "learning_rate": 4.834116786912897e-05,
      "log_odds_chosen": 0.22876164317131042,
      "log_odds_ratio": -0.6974906325340271,
      "logps/chosen": -1.2567417621612549,
      "logps/rejected": -1.4308439493179321,
      "loss": 39.6728,
      "nll_loss": 1.2350414991378784,
      "po_loss": 0.0047349645756185055,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.6283708810806274,
      "rewards/margins": 0.08705107122659683,
      "rewards/rejected": -0.7154219746589661,
      "step": 65
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 66.75170135498047,
      "learning_rate": 4.7808435099299045e-05,
      "log_odds_chosen": 0.30346402525901794,
      "log_odds_ratio": -0.6529059410095215,
      "logps/chosen": -1.2017771005630493,
      "logps/rejected": -1.4399967193603516,
      "loss": 39.0953,
      "nll_loss": 1.2220808267593384,
      "po_loss": -0.00035261179436929524,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -0.6008885502815247,
      "rewards/margins": 0.11910974979400635,
      "rewards/rejected": -0.7199983596801758,
      "step": 70
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 98.31887817382812,
      "learning_rate": 4.720545159477922e-05,
      "log_odds_chosen": 0.32944947481155396,
      "log_odds_ratio": -0.6790199875831604,
      "logps/chosen": -1.1649795770645142,
      "logps/rejected": -1.4293115139007568,
      "loss": 37.4294,
      "nll_loss": 1.1911554336547852,
      "po_loss": -0.021487217396497726,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -0.5824897885322571,
      "rewards/margins": 0.13216595351696014,
      "rewards/rejected": -0.7146557569503784,
      "step": 75
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 122.34395599365234,
      "learning_rate": 4.653407456471222e-05,
      "log_odds_chosen": 0.1852332353591919,
      "log_odds_ratio": -0.7068082094192505,
      "logps/chosen": -1.1496782302856445,
      "logps/rejected": -1.2879629135131836,
      "loss": 36.9845,
      "nll_loss": 1.1549251079559326,
      "po_loss": 0.0008416841737926006,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -0.5748391151428223,
      "rewards/margins": 0.06914232671260834,
      "rewards/rejected": -0.6439814567565918,
      "step": 80
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 130.45028686523438,
      "learning_rate": 4.579637187256222e-05,
      "log_odds_chosen": 0.1647026389837265,
      "log_odds_ratio": -0.6966888904571533,
      "logps/chosen": -1.1415808200836182,
      "logps/rejected": -1.2689316272735596,
      "loss": 37.7102,
      "nll_loss": 1.1786143779754639,
      "po_loss": -0.00017101204139180481,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.5707904100418091,
      "rewards/margins": 0.06367532908916473,
      "rewards/rejected": -0.6344658136367798,
      "step": 85
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 87.87841796875,
      "learning_rate": 4.499461566702685e-05,
      "log_odds_chosen": 0.30306681990623474,
      "log_odds_ratio": -0.658152163028717,
      "logps/chosen": -1.0570913553237915,
      "logps/rejected": -1.2784945964813232,
      "loss": 35.9528,
      "nll_loss": 1.1234948635101318,
      "po_loss": 3.171293064951897e-05,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.5285456776618958,
      "rewards/margins": 0.11070160567760468,
      "rewards/rejected": -0.6392472982406616,
      "step": 90
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 43.475040435791016,
      "learning_rate": 4.413127538374411e-05,
      "log_odds_chosen": 0.1689324676990509,
      "log_odds_ratio": -0.7154847979545593,
      "logps/chosen": -1.1330095529556274,
      "logps/rejected": -1.266969084739685,
      "loss": 37.3848,
      "nll_loss": 1.1682692766189575,
      "po_loss": 3.9124452086980455e-06,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -0.5665047764778137,
      "rewards/margins": 0.06697981059551239,
      "rewards/rejected": -0.6334845423698425,
      "step": 95
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 107.6277084350586,
      "learning_rate": 4.320901013934887e-05,
      "log_odds_chosen": 0.2178940325975418,
      "log_odds_ratio": -0.6711462736129761,
      "logps/chosen": -1.1043890714645386,
      "logps/rejected": -1.2629815340042114,
      "loss": 36.1427,
      "nll_loss": 1.1296303272247314,
      "po_loss": -0.0001712349767331034,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5521945357322693,
      "rewards/margins": 0.07929627597332001,
      "rewards/rejected": -0.6314907670021057,
      "step": 100
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 39.917747497558594,
      "learning_rate": 4.223066054130568e-05,
      "log_odds_chosen": 0.27735456824302673,
      "log_odds_ratio": -0.6796360611915588,
      "logps/chosen": -1.0888252258300781,
      "logps/rejected": -1.319602608680725,
      "loss": 36.929,
      "nll_loss": 1.15861976146698,
      "po_loss": -0.004588633310049772,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -0.5444126129150391,
      "rewards/margins": 0.11538869142532349,
      "rewards/rejected": -0.6598013043403625,
      "step": 105
    },
    {
      "epoch": 1.037925925925926,
      "grad_norm": 50.280296325683594,
      "learning_rate": 4.1199239938743797e-05,
      "log_odds_chosen": 0.49575677514076233,
      "log_odds_ratio": -0.5771303772926331,
      "logps/chosen": -0.9713961482048035,
      "logps/rejected": -1.3024492263793945,
      "loss": 28.2864,
      "nll_loss": 0.9926872849464417,
      "po_loss": -0.0036509016063064337,
      "rewards/accuracies": 0.6993007063865662,
      "rewards/chosen": -0.48569807410240173,
      "rewards/margins": 0.16552653908729553,
      "rewards/rejected": -0.6512246131896973,
      "step": 110
    },
    {
      "epoch": 1.0853333333333333,
      "grad_norm": 93.71875762939453,
      "learning_rate": 4.0117925141242174e-05,
      "log_odds_chosen": 0.6905006170272827,
      "log_odds_ratio": -0.5269922018051147,
      "logps/chosen": -0.8827330470085144,
      "logps/rejected": -1.3338351249694824,
      "loss": 30.1718,
      "nll_loss": 0.9528656005859375,
      "po_loss": -0.009996998123824596,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.4413665235042572,
      "rewards/margins": 0.225551038980484,
      "rewards/rejected": -0.6669175624847412,
      "step": 115
    },
    {
      "epoch": 1.1327407407407408,
      "grad_norm": 169.19061279296875,
      "learning_rate": 3.899004663415084e-05,
      "log_odds_chosen": 0.6179796457290649,
      "log_odds_ratio": -0.5969129800796509,
      "logps/chosen": -0.9934114217758179,
      "logps/rejected": -1.424770712852478,
      "loss": 29.7219,
      "nll_loss": 0.9907350540161133,
      "po_loss": -0.061925552785396576,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.49670571088790894,
      "rewards/margins": 0.21567964553833008,
      "rewards/rejected": -0.712385356426239,
      "step": 120
    },
    {
      "epoch": 1.1801481481481482,
      "grad_norm": 924.751708984375,
      "learning_rate": 3.781907832058587e-05,
      "log_odds_chosen": 0.631129264831543,
      "log_odds_ratio": -0.5441834330558777,
      "logps/chosen": -0.9922526478767395,
      "logps/rejected": -1.4152342081069946,
      "loss": 29.274,
      "nll_loss": 0.9811958074569702,
      "po_loss": -0.06638330966234207,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.49612632393836975,
      "rewards/margins": 0.21149075031280518,
      "rewards/rejected": -0.7076171040534973,
      "step": 125
    },
    {
      "epoch": 1.2275555555555555,
      "grad_norm": 268.8498229980469,
      "learning_rate": 3.660862682169282e-05,
      "log_odds_chosen": 1.0107532739639282,
      "log_odds_ratio": -0.5149319171905518,
      "logps/chosen": -0.9752801656723022,
      "logps/rejected": -1.736717939376831,
      "loss": 28.7769,
      "nll_loss": 1.0052934885025024,
      "po_loss": -0.10601623356342316,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4876400828361511,
      "rewards/margins": 0.38071882724761963,
      "rewards/rejected": -0.8683589696884155,
      "step": 130
    },
    {
      "epoch": 1.274962962962963,
      "grad_norm": 125.7737045288086,
      "learning_rate": 3.5362420368134356e-05,
      "log_odds_chosen": 1.326555848121643,
      "log_odds_ratio": -0.5516592264175415,
      "logps/chosen": -1.1946276426315308,
      "logps/rejected": -2.280578374862671,
      "loss": 20.1976,
      "nll_loss": 1.043860912322998,
      "po_loss": -0.4126869738101959,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.5973138213157654,
      "rewards/margins": 0.5429753065109253,
      "rewards/rejected": -1.1402891874313354,
      "step": 135
    },
    {
      "epoch": 1.3223703703703704,
      "grad_norm": 108.3899917602539,
      "learning_rate": 3.408429731701635e-05,
      "log_odds_chosen": 0.807600200176239,
      "log_odds_ratio": -0.6807276010513306,
      "logps/chosen": -1.2439929246902466,
      "logps/rejected": -1.8694818019866943,
      "loss": 26.9588,
      "nll_loss": 1.121119499206543,
      "po_loss": -0.27865657210350037,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.6219964623451233,
      "rewards/margins": 0.3127444386482239,
      "rewards/rejected": -0.9347409009933472,
      "step": 140
    },
    {
      "epoch": 1.3697777777777778,
      "grad_norm": 109.62158966064453,
      "learning_rate": 3.2778194329621104e-05,
      "log_odds_chosen": 0.6288902163505554,
      "log_odds_ratio": -0.5495084524154663,
      "logps/chosen": -0.9818429946899414,
      "logps/rejected": -1.437950611114502,
      "loss": 30.7098,
      "nll_loss": 1.0324475765228271,
      "po_loss": -0.07276667654514313,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.4909214973449707,
      "rewards/margins": 0.22805377840995789,
      "rewards/rejected": -0.718975305557251,
      "step": 145
    },
    {
      "epoch": 1.417185185185185,
      "grad_norm": 76.47566223144531,
      "learning_rate": 3.144813424636031e-05,
      "log_odds_chosen": 0.7733277082443237,
      "log_odds_ratio": -0.517223596572876,
      "logps/chosen": -0.9549392461776733,
      "logps/rejected": -1.517096757888794,
      "loss": 30.5632,
      "nll_loss": 0.9677143096923828,
      "po_loss": -0.0126152653247118,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.47746962308883667,
      "rewards/margins": 0.2810787260532379,
      "rewards/rejected": -0.758548378944397,
      "step": 150
    },
    {
      "epoch": 1.4645925925925927,
      "grad_norm": 287.5429382324219,
      "learning_rate": 3.0098213696293542e-05,
      "log_odds_chosen": 1.0348070859909058,
      "log_odds_ratio": -0.5240525603294373,
      "logps/chosen": -0.9443733096122742,
      "logps/rejected": -1.771649956703186,
      "loss": 28.328,
      "nll_loss": 0.9806075096130371,
      "po_loss": -0.09535732865333557,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.4721866548061371,
      "rewards/margins": 0.41363826394081116,
      "rewards/rejected": -0.885824978351593,
      "step": 155
    },
    {
      "epoch": 1.512,
      "grad_norm": 1569.7928466796875,
      "learning_rate": 2.8732590479375165e-05,
      "log_odds_chosen": 1.672914743423462,
      "log_odds_ratio": -0.5939630270004272,
      "logps/chosen": -1.1455912590026855,
      "logps/rejected": -2.59106707572937,
      "loss": 9.8547,
      "nll_loss": 1.0021655559539795,
      "po_loss": -0.6942048668861389,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.5727956295013428,
      "rewards/margins": 0.7227380275726318,
      "rewards/rejected": -1.295533537864685,
      "step": 160
    },
    {
      "epoch": 1.5594074074074074,
      "grad_norm": 363.3552551269531,
      "learning_rate": 2.7355470760292956e-05,
      "log_odds_chosen": 2.250706434249878,
      "log_odds_ratio": -0.7251700162887573,
      "logps/chosen": -1.5436394214630127,
      "logps/rejected": -3.587456464767456,
      "loss": -2.5921,
      "nll_loss": 1.1975200176239014,
      "po_loss": -1.2785217761993408,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7718197107315063,
      "rewards/margins": 1.0219084024429321,
      "rewards/rejected": -1.793728232383728,
      "step": 165
    },
    {
      "epoch": 1.6068148148148147,
      "grad_norm": 122.50560760498047,
      "learning_rate": 2.597109611334169e-05,
      "log_odds_chosen": 1.8454078435897827,
      "log_odds_ratio": -0.5737333297729492,
      "logps/chosen": -1.1362860202789307,
      "logps/rejected": -2.774848699569702,
      "loss": 18.5002,
      "nll_loss": 1.105724811553955,
      "po_loss": -0.5275925993919373,
      "rewards/accuracies": 0.7718750238418579,
      "rewards/chosen": -0.5681430101394653,
      "rewards/margins": 0.8192813992500305,
      "rewards/rejected": -1.387424349784851,
      "step": 170
    },
    {
      "epoch": 1.6542222222222223,
      "grad_norm": 470.26031494140625,
      "learning_rate": 2.458373045823404e-05,
      "log_odds_chosen": 2.652867078781128,
      "log_odds_ratio": -0.5222973823547363,
      "logps/chosen": -1.2968136072158813,
      "logps/rejected": -3.699096202850342,
      "loss": -21.8498,
      "nll_loss": 1.1843297481536865,
      "po_loss": -1.8671356439590454,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.6484068036079407,
      "rewards/margins": 1.201141357421875,
      "rewards/rejected": -1.849548101425171,
      "step": 175
    },
    {
      "epoch": 1.7016296296296296,
      "grad_norm": 5300.20166015625,
      "learning_rate": 2.3197646927086697e-05,
      "log_odds_chosen": 4.673213005065918,
      "log_odds_ratio": -0.7253745794296265,
      "logps/chosen": -2.2312347888946533,
      "logps/rejected": -6.676938056945801,
      "loss": 13.7486,
      "nll_loss": 1.6908314228057861,
      "po_loss": -1.2611885070800781,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -1.1156173944473267,
      "rewards/margins": 2.222851276397705,
      "rewards/rejected": -3.3384690284729004,
      "step": 180
    },
    {
      "epoch": 1.749037037037037,
      "grad_norm": 7335.25634765625,
      "learning_rate": 2.1817114703032176e-05,
      "log_odds_chosen": 6.68955135345459,
      "log_odds_ratio": -1.325089693069458,
      "logps/chosen": -3.713343858718872,
      "logps/rejected": -10.222986221313477,
      "loss": 73.5102,
      "nll_loss": 2.453530788421631,
      "po_loss": -0.15633538365364075,
      "rewards/accuracies": 0.7718750238418579,
      "rewards/chosen": -1.856671929359436,
      "rewards/margins": 3.254821300506592,
      "rewards/rejected": -5.111493110656738,
      "step": 185
    },
    {
      "epoch": 1.7964444444444445,
      "grad_norm": 2789.342041015625,
      "learning_rate": 2.0446385870993467e-05,
      "log_odds_chosen": 8.35153579711914,
      "log_odds_ratio": -1.271281361579895,
      "logps/chosen": -3.2499496936798096,
      "logps/rejected": -11.391130447387695,
      "loss": 5.3516,
      "nll_loss": 2.1729094982147217,
      "po_loss": -2.0056724548339844,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.6249748468399048,
      "rewards/margins": 4.070590496063232,
      "rewards/rejected": -5.695565223693848,
      "step": 190
    },
    {
      "epoch": 1.8438518518518519,
      "grad_norm": 466.29412841796875,
      "learning_rate": 1.9089682321121834e-05,
      "log_odds_chosen": 6.304251194000244,
      "log_odds_ratio": -1.1286901235580444,
      "logps/chosen": -2.1377434730529785,
      "logps/rejected": -8.211523056030273,
      "loss": -67.2106,
      "nll_loss": 1.6573760509490967,
      "po_loss": -3.757708787918091,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -1.0688717365264893,
      "rewards/margins": 3.0368895530700684,
      "rewards/rejected": -4.105761528015137,
      "step": 195
    },
    {
      "epoch": 1.8912592592592592,
      "grad_norm": 9109.884765625,
      "learning_rate": 1.775118274523545e-05,
      "log_odds_chosen": 7.661096096038818,
      "log_odds_ratio": -3.3484745025634766,
      "logps/chosen": -4.9656147956848145,
      "logps/rejected": -12.398049354553223,
      "loss": 51.4285,
      "nll_loss": 3.3246026039123535,
      "po_loss": -1.7174619436264038,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.4828073978424072,
      "rewards/margins": 3.716217517852783,
      "rewards/rejected": -6.199024677276611,
      "step": 200
    },
    {
      "epoch": 1.9386666666666668,
      "grad_norm": 3225.781982421875,
      "learning_rate": 1.643500976631037e-05,
      "log_odds_chosen": 5.402312278747559,
      "log_odds_ratio": -2.382956027984619,
      "logps/chosen": -4.517752647399902,
      "logps/rejected": -9.684160232543945,
      "loss": 3.3627,
      "nll_loss": 3.054727077484131,
      "po_loss": -2.94964337348938,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.258876323699951,
      "rewards/margins": 2.5832042694091797,
      "rewards/rejected": -4.842080116271973,
      "step": 205
    },
    {
      "epoch": 1.986074074074074,
      "grad_norm": 14159.1806640625,
      "learning_rate": 1.514521724066537e-05,
      "log_odds_chosen": 4.498274803161621,
      "log_odds_ratio": -2.2259490489959717,
      "logps/chosen": -4.709282875061035,
      "logps/rejected": -8.998777389526367,
      "loss": 44.9858,
      "nll_loss": 3.0146031379699707,
      "po_loss": -1.6087970733642578,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.3546414375305176,
      "rewards/margins": 2.144747257232666,
      "rewards/rejected": -4.499388694763184,
      "step": 210
    },
    {
      "epoch": 2.0284444444444443,
      "grad_norm": 7633.78271484375,
      "learning_rate": 1.3885777771950348e-05,
      "log_odds_chosen": 5.469094276428223,
      "log_odds_ratio": -1.9151403903961182,
      "logps/chosen": -3.9436492919921875,
      "logps/rejected": -9.1085844039917,
      "loss": -46.6478,
      "nll_loss": 2.75101900100708,
      "po_loss": -4.382059574127197,
      "rewards/accuracies": 0.7692307829856873,
      "rewards/chosen": -1.9718246459960938,
      "rewards/margins": 2.582467794418335,
      "rewards/rejected": -4.55429220199585,
      "step": 215
    },
    {
      "epoch": 2.075851851851852,
      "grad_norm": 11660.62109375,
      "learning_rate": 1.2660570475395683e-05,
      "log_odds_chosen": 5.316206932067871,
      "log_odds_ratio": -1.9996505975723267,
      "logps/chosen": -4.677931785583496,
      "logps/rejected": -9.788928985595703,
      "loss": 15.4857,
      "nll_loss": 3.0612142086029053,
      "po_loss": -2.577287197113037,
      "rewards/accuracies": 0.7406250238418579,
      "rewards/chosen": -2.338965892791748,
      "rewards/margins": 2.5554988384246826,
      "rewards/rejected": -4.894464492797852,
      "step": 220
    },
    {
      "epoch": 2.1232592592592594,
      "grad_norm": 8261.25,
      "learning_rate": 1.1473369030008974e-05,
      "log_odds_chosen": 6.4358320236206055,
      "log_odds_ratio": -1.4537756443023682,
      "logps/chosen": -3.936769485473633,
      "logps/rejected": -10.128541946411133,
      "loss": -119.1289,
      "nll_loss": 2.613402843475342,
      "po_loss": -6.336179733276367,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.9683847427368164,
      "rewards/margins": 3.09588623046875,
      "rewards/rejected": -5.064270973205566,
      "step": 225
    },
    {
      "epoch": 2.1706666666666665,
      "grad_norm": 8660.0673828125,
      "learning_rate": 1.0327830055518842e-05,
      "log_odds_chosen": 6.7201948165893555,
      "log_odds_ratio": -1.9511137008666992,
      "logps/chosen": -6.028278350830078,
      "logps/rejected": -12.52048110961914,
      "loss": 111.5082,
      "nll_loss": 3.882694959640503,
      "po_loss": -0.3980643153190613,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -3.014139175415039,
      "rewards/margins": 3.2461013793945312,
      "rewards/rejected": -6.26024055480957,
      "step": 230
    },
    {
      "epoch": 2.218074074074074,
      "grad_norm": 4395.22802734375,
      "learning_rate": 9.227481849865235e-06,
      "log_odds_chosen": 4.666752815246582,
      "log_odds_ratio": -1.0462980270385742,
      "logps/chosen": -3.0177314281463623,
      "logps/rejected": -7.436758995056152,
      "loss": -53.0945,
      "nll_loss": 2.226374626159668,
      "po_loss": -3.8855788707733154,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -1.5088657140731812,
      "rewards/margins": 2.2095141410827637,
      "rewards/rejected": -3.718379497528076,
      "step": 235
    },
    {
      "epoch": 2.2654814814814817,
      "grad_norm": 14886.419921875,
      "learning_rate": 8.175713521924978e-06,
      "log_odds_chosen": 6.242539882659912,
      "log_odds_ratio": -1.3734468221664429,
      "logps/chosen": -2.750431537628174,
      "logps/rejected": -8.688729286193848,
      "loss": -111.1189,
      "nll_loss": 2.0052638053894043,
      "po_loss": -5.4777302742004395,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.375215768814087,
      "rewards/margins": 2.969148635864258,
      "rewards/rejected": -4.344364643096924,
      "step": 240
    },
    {
      "epoch": 2.3128888888888888,
      "grad_norm": 4024.9541015625,
      "learning_rate": 7.1757645529443665e-06,
      "log_odds_chosen": 7.824398994445801,
      "log_odds_ratio": -0.8879048228263855,
      "logps/chosen": -2.6384353637695312,
      "logps/rejected": -10.153287887573242,
      "loss": -210.0702,
      "nll_loss": 1.8768306970596313,
      "po_loss": -8.441526412963867,
      "rewards/accuracies": 0.815625011920929,
      "rewards/chosen": -1.3192176818847656,
      "rewards/margins": 3.7574257850646973,
      "rewards/rejected": -5.076643943786621,
      "step": 245
    },
    {
      "epoch": 2.3602962962962963,
      "grad_norm": 3342.098388671875,
      "learning_rate": 6.230714818829733e-06,
      "log_odds_chosen": 6.20465087890625,
      "log_odds_ratio": -0.8402653932571411,
      "logps/chosen": -1.9506229162216187,
      "logps/rejected": -7.861325740814209,
      "loss": -120.2652,
      "nll_loss": 1.5267200469970703,
      "po_loss": -5.285006999969482,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.9753114581108093,
      "rewards/margins": 2.9553513526916504,
      "rewards/rejected": -3.9306628704071045,
      "step": 250
    },
    {
      "epoch": 2.407703703703704,
      "grad_norm": 3261.58203125,
      "learning_rate": 5.343475104027743e-06,
      "log_odds_chosen": 5.790114879608154,
      "log_odds_ratio": -1.1742393970489502,
      "logps/chosen": -2.963078022003174,
      "logps/rejected": -8.470439910888672,
      "loss": -72.0198,
      "nll_loss": 1.9134031534194946,
      "po_loss": -4.164022445678711,
      "rewards/accuracies": 0.8031250238418579,
      "rewards/chosen": -1.481539011001587,
      "rewards/margins": 2.753680944442749,
      "rewards/rejected": -4.235219955444336,
      "step": 255
    },
    {
      "epoch": 2.455111111111111,
      "grad_norm": 2722.1357421875,
      "learning_rate": 4.516778136213037e-06,
      "log_odds_chosen": 6.029698848724365,
      "log_odds_ratio": -0.8935467600822449,
      "logps/chosen": -2.35087513923645,
      "logps/rejected": -8.128339767456055,
      "loss": -95.4577,
      "nll_loss": 1.7904222011566162,
      "po_loss": -4.773475646972656,
      "rewards/accuracies": 0.7906249761581421,
      "rewards/chosen": -1.175437569618225,
      "rewards/margins": 2.888732433319092,
      "rewards/rejected": -4.064169883728027,
      "step": 260
    },
    {
      "epoch": 2.5025185185185186,
      "grad_norm": 1122.66064453125,
      "learning_rate": 3.7531701693965554e-06,
      "log_odds_chosen": 7.004757881164551,
      "log_odds_ratio": -0.9651885032653809,
      "logps/chosen": -2.6934361457824707,
      "logps/rejected": -9.452727317810059,
      "loss": -36.8828,
      "nll_loss": 1.900667428970337,
      "po_loss": -3.0532543659210205,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3467180728912354,
      "rewards/margins": 3.379645586013794,
      "rewards/rejected": -4.726363658905029,
      "step": 265
    },
    {
      "epoch": 2.549925925925926,
      "grad_norm": 3068.10888671875,
      "learning_rate": 3.055003141378948e-06,
      "log_odds_chosen": 7.71005916595459,
      "log_odds_ratio": -0.8755048513412476,
      "logps/chosen": -2.4941697120666504,
      "logps/rejected": -9.900228500366211,
      "loss": -182.7595,
      "nll_loss": 1.8155403137207031,
      "po_loss": -7.526776313781738,
      "rewards/accuracies": 0.784375011920929,
      "rewards/chosen": -1.2470848560333252,
      "rewards/margins": 3.7030296325683594,
      "rewards/rejected": -4.9501142501831055,
      "step": 270
    },
    {
      "epoch": 2.5973333333333333,
      "grad_norm": 1143.6956787109375,
      "learning_rate": 2.424427429704365e-06,
      "log_odds_chosen": 7.963990688323975,
      "log_odds_ratio": -1.225984811782837,
      "logps/chosen": -2.6055846214294434,
      "logps/rejected": -10.264480590820312,
      "loss": -156.3998,
      "nll_loss": 1.759436845779419,
      "po_loss": -6.6469316482543945,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -1.3027923107147217,
      "rewards/margins": 3.8294479846954346,
      "rewards/rejected": -5.132240295410156,
      "step": 275
    },
    {
      "epoch": 2.644740740740741,
      "grad_norm": 2151.653564453125,
      "learning_rate": 1.8633852284264508e-06,
      "log_odds_chosen": 9.631061553955078,
      "log_odds_ratio": -0.9474805593490601,
      "logps/chosen": -2.6136832237243652,
      "logps/rejected": -11.93617057800293,
      "loss": -229.753,
      "nll_loss": 1.9048538208007812,
      "po_loss": -9.084635734558105,
      "rewards/accuracies": 0.7906249761581421,
      "rewards/chosen": -1.3068416118621826,
      "rewards/margins": 4.661242961883545,
      "rewards/rejected": -5.968085289001465,
      "step": 280
    },
    {
      "epoch": 2.6921481481481484,
      "grad_norm": 8310.8056640625,
      "learning_rate": 1.3736045660864034e-06,
      "log_odds_chosen": 7.31323766708374,
      "log_odds_ratio": -1.1464475393295288,
      "logps/chosen": -2.9281296730041504,
      "logps/rejected": -9.972752571105957,
      "loss": -138.1873,
      "nll_loss": 2.1238648891448975,
      "po_loss": -6.442216396331787,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -1.4640648365020752,
      "rewards/margins": 3.5223114490509033,
      "rewards/rejected": -4.9863762855529785,
      "step": 285
    },
    {
      "epoch": 2.7395555555555555,
      "grad_norm": 8654.1640625,
      "learning_rate": 9.565939833279192e-07,
      "log_odds_chosen": 6.848095893859863,
      "log_odds_ratio": -1.5750070810317993,
      "logps/chosen": -3.325796127319336,
      "logps/rejected": -9.924636840820312,
      "loss": -32.2877,
      "nll_loss": 2.305940628051758,
      "po_loss": -3.3149306774139404,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.662898063659668,
      "rewards/margins": 3.299420118331909,
      "rewards/rejected": -4.962318420410156,
      "step": 290
    },
    {
      "epoch": 2.786962962962963,
      "grad_norm": 5190.52734375,
      "learning_rate": 6.136378865420872e-07,
      "log_odds_chosen": 7.09508752822876,
      "log_odds_ratio": -1.0497550964355469,
      "logps/chosen": -2.6469645500183105,
      "logps/rejected": -9.480436325073242,
      "loss": -183.1275,
      "nll_loss": 1.8889389038085938,
      "po_loss": -7.611671447753906,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -1.3234822750091553,
      "rewards/margins": 3.416736125946045,
      "rewards/rejected": -4.740218162536621,
      "step": 295
    },
    {
      "epoch": 2.83437037037037,
      "grad_norm": 5004.75390625,
      "learning_rate": 3.45792591853214e-07,
      "log_odds_chosen": 8.076799392700195,
      "log_odds_ratio": -1.2268081903457642,
      "logps/chosen": -2.7893056869506836,
      "logps/rejected": -10.587810516357422,
      "loss": -172.2458,
      "nll_loss": 1.945686936378479,
      "po_loss": -7.328370094299316,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.3946528434753418,
      "rewards/margins": 3.899252414703369,
      "rewards/rejected": -5.293905258178711,
      "step": 300
    },
    {
      "epoch": 2.8817777777777778,
      "grad_norm": 1290.5986328125,
      "learning_rate": 1.538830716302092e-07,
      "log_odds_chosen": 8.941948890686035,
      "log_odds_ratio": -0.9725468754768372,
      "logps/chosen": -2.6866989135742188,
      "logps/rejected": -11.317400932312012,
      "loss": -192.9953,
      "nll_loss": 1.9194834232330322,
      "po_loss": -7.95058536529541,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -1.3433494567871094,
      "rewards/margins": 4.315350532531738,
      "rewards/rejected": -5.658700466156006,
      "step": 305
    },
    {
      "epoch": 2.9291851851851853,
      "grad_norm": 1017.9197387695312,
      "learning_rate": 3.8500413544415025e-08,
      "log_odds_chosen": 9.571866989135742,
      "log_odds_ratio": -1.065990686416626,
      "logps/chosen": -2.9964630603790283,
      "logps/rejected": -12.271339416503906,
      "loss": -196.4677,
      "nll_loss": 2.1196494102478027,
      "po_loss": -8.25926399230957,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.4982315301895142,
      "rewards/margins": 4.637437343597412,
      "rewards/rejected": -6.135669708251953,
      "step": 310
    },
    {
      "epoch": 2.9765925925925925,
      "grad_norm": 4958.69287109375,
      "learning_rate": 0.0,
      "log_odds_chosen": 9.563053131103516,
      "log_odds_ratio": -0.80206698179245,
      "logps/chosen": -2.0710840225219727,
      "logps/rejected": -11.277410507202148,
      "loss": -302.3181,
      "nll_loss": 1.6443132162094116,
      "po_loss": -11.091754913330078,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.0355420112609863,
      "rewards/margins": 4.603163719177246,
      "rewards/rejected": -5.638705253601074,
      "step": 315
    },
    {
      "epoch": 2.9765925925925925,
      "step": 315,
      "total_flos": 0.0,
      "train_loss": 23.796105109320745,
      "train_runtime": 9699.8845,
      "train_samples_per_second": 2.088,
      "train_steps_per_second": 0.032
    }
  ],
  "logging_steps": 5,
  "max_steps": 315,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}