{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9993196353245339,
  "eval_steps": 1000,
  "global_step": 459,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 2.1739130434782606e-08,
      "logits/chosen": -2.3381757736206055,
      "logits/rejected": -2.3721675872802734,
      "logps/chosen": -308.3559875488281,
      "logps/rejected": -268.3885192871094,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/mix_margin": -1.0728836485895954e-08,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 2.1739130434782607e-07,
      "logits/chosen": -2.3499717712402344,
      "logits/rejected": -2.30268931388855,
      "logps/chosen": -347.79754638671875,
      "logps/rejected": -304.29705810546875,
      "loss": 0.6927,
      "rewards/accuracies": 0.4513888955116272,
      "rewards/chosen": 0.0014620342990383506,
      "rewards/confidence": -0.011617942713201046,
      "rewards/confidence_mean_diff": 0.011617942713201046,
      "rewards/confidence_moving_diff": 0.0008721869089640677,
      "rewards/margins": 0.0018856121459975839,
      "rewards/mix_margin": 0.000961379031650722,
      "rewards/real_percentage": 14.571428298950195,
      "rewards/rejected": -0.00042357799247838557,
      "step": 10
    },
    {
      "epoch": 0.04,
      "learning_rate": 4.3478260869565214e-07,
      "logits/chosen": -2.4307799339294434,
      "logits/rejected": -2.4313855171203613,
      "logps/chosen": -337.871826171875,
      "logps/rejected": -309.18695068359375,
      "loss": 0.6902,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.018211863934993744,
      "rewards/confidence": -0.013249741867184639,
      "rewards/confidence_mean_diff": 0.013249741867184639,
      "rewards/confidence_moving_diff": 0.0012787743471562862,
      "rewards/margins": 0.004662538878619671,
      "rewards/mix_margin": 0.002308625727891922,
      "rewards/real_percentage": 8.800000190734863,
      "rewards/rejected": 0.013549325987696648,
      "step": 20
    },
    {
      "epoch": 0.07,
      "learning_rate": 6.521739130434782e-07,
      "logits/chosen": -2.3925347328186035,
      "logits/rejected": -2.3656105995178223,
      "logps/chosen": -333.0788879394531,
      "logps/rejected": -297.11865234375,
      "loss": 0.679,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.07145825028419495,
      "rewards/confidence": -0.036696191877126694,
      "rewards/confidence_mean_diff": 0.036696191877126694,
      "rewards/confidence_moving_diff": 0.0040471795946359634,
      "rewards/margins": 0.04734768718481064,
      "rewards/mix_margin": 0.024542126804590225,
      "rewards/real_percentage": 7.400000095367432,
      "rewards/rejected": 0.02411056123673916,
      "step": 30
    },
    {
      "epoch": 0.09,
      "learning_rate": 8.695652173913043e-07,
      "logits/chosen": -2.276813507080078,
      "logits/rejected": -2.2625725269317627,
      "logps/chosen": -302.2676696777344,
      "logps/rejected": -285.21478271484375,
      "loss": 0.6483,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.041187357157468796,
      "rewards/confidence": -0.08288822323083878,
      "rewards/confidence_mean_diff": 0.08288822323083878,
      "rewards/confidence_moving_diff": 0.0016981459921225905,
      "rewards/margins": 0.11979661136865616,
      "rewards/mix_margin": 0.06444107741117477,
      "rewards/real_percentage": 5.599999904632568,
      "rewards/rejected": -0.07860924303531647,
      "step": 40
    },
    {
      "epoch": 0.11,
      "learning_rate": 9.997685665512417e-07,
      "logits/chosen": -2.103567600250244,
      "logits/rejected": -2.0963990688323975,
      "logps/chosen": -310.5840759277344,
      "logps/rejected": -280.5057373046875,
      "loss": 0.6262,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07748699188232422,
      "rewards/confidence": -0.15422135591506958,
      "rewards/confidence_mean_diff": 0.15422135591506958,
      "rewards/confidence_moving_diff": 0.007699769921600819,
      "rewards/margins": 0.17081865668296814,
      "rewards/mix_margin": 0.09792973846197128,
      "rewards/real_percentage": 6.599999904632568,
      "rewards/rejected": -0.24830564856529236,
      "step": 50
    },
    {
      "epoch": 0.13,
      "learning_rate": 9.971674001050686e-07,
      "logits/chosen": -1.7708213329315186,
      "logits/rejected": -1.7266209125518799,
      "logps/chosen": -368.86602783203125,
      "logps/rejected": -356.1971740722656,
      "loss": 0.5962,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.36815953254699707,
      "rewards/confidence": -0.21500273048877716,
      "rewards/confidence_mean_diff": 0.21500273048877716,
      "rewards/confidence_moving_diff": 0.00331355188973248,
      "rewards/margins": 0.3264981508255005,
      "rewards/mix_margin": 0.19259774684906006,
      "rewards/real_percentage": 6.800000190734863,
      "rewards/rejected": -0.6946576833724976,
      "step": 60
    },
    {
      "epoch": 0.15,
      "learning_rate": 9.916908694464492e-07,
      "logits/chosen": -1.764725685119629,
      "logits/rejected": -1.739040732383728,
      "logps/chosen": -355.16314697265625,
      "logps/rejected": -341.5367126464844,
      "loss": 0.5938,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7925843000411987,
      "rewards/confidence": -0.2245199978351593,
      "rewards/confidence_mean_diff": 0.2245199978351593,
      "rewards/confidence_moving_diff": 0.017536107450723648,
      "rewards/margins": 0.3716357350349426,
      "rewards/mix_margin": 0.21924924850463867,
      "rewards/real_percentage": 5.800000190734863,
      "rewards/rejected": -1.1642199754714966,
      "step": 70
    },
    {
      "epoch": 0.17,
      "learning_rate": 9.833706480524877e-07,
      "logits/chosen": -1.933165192604065,
      "logits/rejected": -1.9561519622802734,
      "logps/chosen": -402.5296325683594,
      "logps/rejected": -389.52899169921875,
      "loss": 0.5797,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.0171725749969482,
      "rewards/confidence": -0.4024506211280823,
      "rewards/confidence_mean_diff": 0.4024506211280823,
      "rewards/confidence_moving_diff": -0.013792142271995544,
      "rewards/margins": 0.32845553755760193,
      "rewards/mix_margin": 0.17139557003974915,
      "rewards/real_percentage": 6.199999809265137,
      "rewards/rejected": -1.345628023147583,
      "step": 80
    },
    {
      "epoch": 0.2,
      "learning_rate": 9.722548558659455e-07,
      "logits/chosen": -1.8798820972442627,
      "logits/rejected": -1.87497878074646,
      "logps/chosen": -400.71405029296875,
      "logps/rejected": -409.4770812988281,
      "loss": 0.5751,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.0970375537872314,
      "rewards/confidence": -0.29926934838294983,
      "rewards/confidence_mean_diff": 0.29926934838294983,
      "rewards/confidence_moving_diff": 0.015591544099152088,
      "rewards/margins": 0.3549976646900177,
      "rewards/mix_margin": 0.20659112930297852,
      "rewards/real_percentage": 6.800000190734863,
      "rewards/rejected": -1.4520353078842163,
      "step": 90
    },
    {
      "epoch": 0.22,
      "learning_rate": 9.584077809938854e-07,
      "logits/chosen": -1.9064958095550537,
      "logits/rejected": -1.8910179138183594,
      "logps/chosen": -380.4066467285156,
      "logps/rejected": -376.3758850097656,
      "loss": 0.5538,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.8957662582397461,
      "rewards/confidence": -0.21787028014659882,
      "rewards/confidence_mean_diff": 0.21787028014659882,
      "rewards/confidence_moving_diff": -0.01832030713558197,
      "rewards/margins": 0.5051447749137878,
      "rewards/mix_margin": 0.28336718678474426,
      "rewards/real_percentage": 5.599999904632568,
      "rewards/rejected": -1.4009110927581787,
      "step": 100
    },
    {
      "epoch": 0.24,
      "learning_rate": 9.419095078978505e-07,
      "logits/chosen": -1.9809643030166626,
      "logits/rejected": -1.9503233432769775,
      "logps/chosen": -386.52081298828125,
      "logps/rejected": -377.4222717285156,
      "loss": 0.5741,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7311219573020935,
      "rewards/confidence": -0.30758336186408997,
      "rewards/confidence_mean_diff": 0.30758336186408997,
      "rewards/confidence_moving_diff": 0.007133707404136658,
      "rewards/margins": 0.41949161887168884,
      "rewards/mix_margin": 0.2570333480834961,
      "rewards/real_percentage": 6.800000190734863,
      "rewards/rejected": -1.15061354637146,
      "step": 110
    },
    {
      "epoch": 0.26,
      "learning_rate": 9.228554542259359e-07,
      "logits/chosen": -2.112670421600342,
      "logits/rejected": -2.097276210784912,
      "logps/chosen": -356.1893615722656,
      "logps/rejected": -397.5374450683594,
      "loss": 0.5385,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.8588116765022278,
      "rewards/confidence": -0.3459985852241516,
      "rewards/confidence_mean_diff": 0.3459985852241516,
      "rewards/confidence_moving_diff": 0.003056983696296811,
      "rewards/margins": 0.68636155128479,
      "rewards/mix_margin": 0.2696496844291687,
      "rewards/real_percentage": 6.400000095367432,
      "rewards/rejected": -1.545173168182373,
      "step": 120
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.013558189654817e-07,
      "logits/chosen": -1.9888393878936768,
      "logits/rejected": -1.9572227001190186,
      "logps/chosen": -455.81561279296875,
      "logps/rejected": -461.1678161621094,
      "loss": 0.5316,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.197947382926941,
      "rewards/confidence": -0.27709904313087463,
      "rewards/confidence_mean_diff": 0.27709904313087463,
      "rewards/confidence_moving_diff": 0.008776068687438965,
      "rewards/margins": 0.6648138165473938,
      "rewards/mix_margin": 0.34475114941596985,
      "rewards/real_percentage": 5.400000095367432,
      "rewards/rejected": -1.86276113986969,
      "step": 130
    },
    {
      "epoch": 0.3,
      "learning_rate": 8.775349451079948e-07,
      "logits/chosen": -2.085186004638672,
      "logits/rejected": -2.0593903064727783,
      "logps/chosen": -428.7594299316406,
      "logps/rejected": -433.14324951171875,
      "loss": 0.536,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.1666946411132812,
      "rewards/confidence": -0.3819279074668884,
      "rewards/confidence_mean_diff": 0.3819279074668884,
      "rewards/confidence_moving_diff": -0.0010703206062316895,
      "rewards/margins": 0.5809696912765503,
      "rewards/mix_margin": 0.281490296125412,
      "rewards/real_percentage": 5.199999809265137,
      "rewards/rejected": -1.747664451599121,
      "step": 140
    },
    {
      "epoch": 0.33,
      "learning_rate": 8.515306005123179e-07,
      "logits/chosen": -2.1095032691955566,
      "logits/rejected": -2.10186505317688,
      "logps/chosen": -407.43963623046875,
      "logps/rejected": -430.080322265625,
      "loss": 0.5424,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.9244325160980225,
      "rewards/confidence": -0.3746878504753113,
      "rewards/confidence_mean_diff": 0.3746878504753113,
      "rewards/confidence_moving_diff": 0.001715996884740889,
      "rewards/margins": 0.4158695340156555,
      "rewards/mix_margin": 0.18484942615032196,
      "rewards/real_percentage": 5.800000190734863,
      "rewards/rejected": -2.340301990509033,
      "step": 150
    },
    {
      "epoch": 0.35,
      "learning_rate": 8.234931811251738e-07,
      "logits/chosen": -2.0886027812957764,
      "logits/rejected": -2.0623221397399902,
      "logps/chosen": -434.42889404296875,
      "logps/rejected": -436.99591064453125,
      "loss": 0.5399,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.801783561706543,
      "rewards/confidence": -0.35509809851646423,
      "rewards/confidence_mean_diff": 0.35509809851646423,
      "rewards/confidence_moving_diff": -0.013272729702293873,
      "rewards/margins": 0.5741153955459595,
      "rewards/mix_margin": 0.3070274591445923,
      "rewards/real_percentage": 6.400000095367432,
      "rewards/rejected": -2.375898838043213,
      "step": 160
    },
    {
      "epoch": 0.37,
      "learning_rate": 7.935848411672499e-07,
      "logits/chosen": -2.0530495643615723,
      "logits/rejected": -2.0262792110443115,
      "logps/chosen": -421.6261291503906,
      "logps/rejected": -443.7255859375,
      "loss": 0.5362,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.8145627975463867,
      "rewards/confidence": -0.3444231450557709,
      "rewards/confidence_mean_diff": 0.3444231450557709,
      "rewards/confidence_moving_diff": 0.011829972267150879,
      "rewards/margins": 0.5010958909988403,
      "rewards/mix_margin": 0.2390981912612915,
      "rewards/real_percentage": 6.199999809265137,
      "rewards/rejected": -2.3156585693359375,
      "step": 170
    },
    {
      "epoch": 0.39,
      "learning_rate": 7.619785553153864e-07,
      "logits/chosen": -1.9314157962799072,
      "logits/rejected": -1.9014371633529663,
      "logps/chosen": -421.787353515625,
      "logps/rejected": -416.24273681640625,
      "loss": 0.5448,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.6310911178588867,
      "rewards/confidence": -0.34312066435813904,
      "rewards/confidence_mean_diff": 0.34312066435813904,
      "rewards/confidence_moving_diff": -0.008907085284590721,
      "rewards/margins": 0.6062546968460083,
      "rewards/mix_margin": 0.31984299421310425,
      "rewards/real_percentage": 6.0,
      "rewards/rejected": -2.2373456954956055,
      "step": 180
    },
    {
      "epoch": 0.41,
      "learning_rate": 7.288571183047321e-07,
      "logits/chosen": -1.8551080226898193,
      "logits/rejected": -1.8303585052490234,
      "logps/chosen": -430.4051208496094,
      "logps/rejected": -433.05743408203125,
      "loss": 0.5427,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7081422805786133,
      "rewards/confidence": -0.3352188169956207,
      "rewards/confidence_mean_diff": 0.3352188169956207,
      "rewards/confidence_moving_diff": 0.016140591353178024,
      "rewards/margins": 0.6280410289764404,
      "rewards/mix_margin": 0.3198699355125427,
      "rewards/real_percentage": 6.199999809265137,
      "rewards/rejected": -2.3361833095550537,
      "step": 190
    },
    {
      "epoch": 0.44,
      "learning_rate": 6.944120877366604e-07,
      "logits/chosen": -1.8757364749908447,
      "logits/rejected": -1.862011194229126,
      "logps/chosen": -449.3797302246094,
      "logps/rejected": -497.9132385253906,
      "loss": 0.5307,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.089892864227295,
      "rewards/confidence": -0.4818331301212311,
      "rewards/confidence_mean_diff": 0.4818331301212311,
      "rewards/confidence_moving_diff": 0.015796076506376266,
      "rewards/margins": 0.6883096694946289,
      "rewards/mix_margin": 0.28093141317367554,
      "rewards/real_percentage": 6.400000095367432,
      "rewards/rejected": -2.778202533721924,
      "step": 200
    },
    {
      "epoch": 0.46,
      "learning_rate": 6.588426762067099e-07,
      "logits/chosen": -1.7704166173934937,
      "logits/rejected": -1.7364553213119507,
      "logps/chosen": -506.0335998535156,
      "logps/rejected": -564.422607421875,
      "loss": 0.5411,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.170118570327759,
      "rewards/confidence": -0.4565798342227936,
      "rewards/confidence_mean_diff": 0.4565798342227936,
      "rewards/confidence_moving_diff": -0.0030906558968126774,
      "rewards/margins": 0.7355421781539917,
      "rewards/mix_margin": 0.2624685764312744,
      "rewards/real_percentage": 6.0,
      "rewards/rejected": -2.905660390853882,
      "step": 210
    },
    {
      "epoch": 0.48,
      "learning_rate": 6.223545991599182e-07,
      "logits/chosen": -1.8121325969696045,
      "logits/rejected": -1.7733787298202515,
      "logps/chosen": -460.2236328125,
      "logps/rejected": -474.4139709472656,
      "loss": 0.5638,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9526455402374268,
      "rewards/confidence": -0.44291096925735474,
      "rewards/confidence_mean_diff": 0.44291096925735474,
      "rewards/confidence_moving_diff": -0.026620090007781982,
      "rewards/margins": 0.5833476185798645,
      "rewards/mix_margin": 0.28964775800704956,
      "rewards/real_percentage": 5.599999904632568,
      "rewards/rejected": -2.5359930992126465,
      "step": 220
    },
    {
      "epoch": 0.5,
      "learning_rate": 5.85158885136973e-07,
      "logits/chosen": -1.9277960062026978,
      "logits/rejected": -1.8827558755874634,
      "logps/chosen": -423.71197509765625,
      "logps/rejected": -446.4361267089844,
      "loss": 0.5302,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8649892807006836,
      "rewards/confidence": -0.3467513918876648,
      "rewards/confidence_mean_diff": 0.3467513918876648,
      "rewards/confidence_moving_diff": -0.005762821529060602,
      "rewards/margins": 0.5457550883293152,
      "rewards/mix_margin": 0.22559857368469238,
      "rewards/real_percentage": 5.400000095367432,
      "rewards/rejected": -2.4107441902160645,
      "step": 230
    },
    {
      "epoch": 0.52,
      "learning_rate": 5.474706552921074e-07,
      "logits/chosen": -1.935349464416504,
      "logits/rejected": -1.8973939418792725,
      "logps/chosen": -395.6863708496094,
      "logps/rejected": -423.58782958984375,
      "loss": 0.5307,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.5539497137069702,
      "rewards/confidence": -0.293173611164093,
      "rewards/confidence_mean_diff": 0.293173611164093,
      "rewards/confidence_moving_diff": -0.00015986264043021947,
      "rewards/margins": 0.6723454594612122,
      "rewards/mix_margin": 0.30846795439720154,
      "rewards/real_percentage": 6.199999809265137,
      "rewards/rejected": -2.226295232772827,
      "step": 240
    },
    {
      "epoch": 0.54,
      "learning_rate": 5.095078792413976e-07,
      "logits/chosen": -1.9067703485488892,
      "logits/rejected": -1.8853263854980469,
      "logps/chosen": -412.2330627441406,
      "logps/rejected": -446.505859375,
      "loss": 0.5122,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.381047010421753,
      "rewards/confidence": -0.4467477798461914,
      "rewards/confidence_mean_diff": 0.4467477798461914,
      "rewards/confidence_moving_diff": 0.001344837248325348,
      "rewards/margins": 0.7658092975616455,
      "rewards/mix_margin": 0.30207857489585876,
      "rewards/real_percentage": 7.0,
      "rewards/rejected": -2.1468563079833984,
      "step": 250
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.714901144369981e-07,
      "logits/chosen": -1.9301033020019531,
      "logits/rejected": -1.8551814556121826,
      "logps/chosen": -422.6144104003906,
      "logps/rejected": -459.46405029296875,
      "loss": 0.5135,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.9298696517944336,
      "rewards/confidence": -0.27530989050865173,
      "rewards/confidence_mean_diff": 0.27530989050865173,
      "rewards/confidence_moving_diff": -0.011671778745949268,
      "rewards/margins": 0.7381618618965149,
      "rewards/mix_margin": 0.3380785584449768,
      "rewards/real_percentage": 6.199999809265137,
      "rewards/rejected": -2.668031930923462,
      "step": 260
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.33637236358139e-07,
      "logits/chosen": -1.9588849544525146,
      "logits/rejected": -1.9613529443740845,
      "logps/chosen": -454.5,
      "logps/rejected": -473.86376953125,
      "loss": 0.5282,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.8807973861694336,
      "rewards/confidence": -0.46479693055152893,
      "rewards/confidence_mean_diff": 0.46479693055152893,
      "rewards/confidence_moving_diff": 0.034666456282138824,
      "rewards/margins": 0.520012378692627,
      "rewards/mix_margin": 0.2387557029724121,
      "rewards/real_percentage": 7.0,
      "rewards/rejected": -2.4008097648620605,
      "step": 270
    },
    {
      "epoch": 0.61,
      "learning_rate": 3.9616816686281635e-07,
      "logits/chosen": -1.82375168800354,
      "logits/rejected": -1.8041833639144897,
      "logps/chosen": -437.76922607421875,
      "logps/rejected": -447.7157287597656,
      "loss": 0.4995,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.6059573888778687,
      "rewards/confidence": -0.39664340019226074,
      "rewards/confidence_mean_diff": 0.39664340019226074,
      "rewards/confidence_moving_diff": -0.011675971560180187,
      "rewards/margins": 0.5647900104522705,
      "rewards/mix_margin": 0.2936910390853882,
      "rewards/real_percentage": 6.199999809265137,
      "rewards/rejected": -2.1707472801208496,
      "step": 280
    },
    {
      "epoch": 0.63,
      "learning_rate": 3.592996080547438e-07,
      "logits/chosen": -1.786231279373169,
      "logits/rejected": -1.7279274463653564,
      "logps/chosen": -411.953125,
      "logps/rejected": -432.6134338378906,
      "loss": 0.5355,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9004911184310913,
      "rewards/confidence": -0.3363880217075348,
      "rewards/confidence_mean_diff": 0.3363880217075348,
      "rewards/confidence_moving_diff": 0.010175123810768127,
      "rewards/margins": 0.7472087144851685,
      "rewards/mix_margin": 0.35827359557151794,
      "rewards/real_percentage": 6.800000190734863,
      "rewards/rejected": -2.6476995944976807,
      "step": 290
    },
    {
      "epoch": 0.65,
      "learning_rate": 3.23244788988237e-07,
      "logits/chosen": -1.7489144802093506,
      "logits/rejected": -1.6973133087158203,
      "logps/chosen": -410.25103759765625,
      "logps/rejected": -413.9371032714844,
      "loss": 0.4997,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.2288432121276855,
      "rewards/confidence": -0.4106927514076233,
      "rewards/confidence_mean_diff": 0.4106927514076233,
      "rewards/confidence_moving_diff": 0.02881682850420475,
      "rewards/margins": 0.6875227689743042,
      "rewards/mix_margin": 0.32858434319496155,
      "rewards/real_percentage": 6.400000095367432,
      "rewards/rejected": -2.9163661003112793,
      "step": 300
    },
    {
      "epoch": 0.67,
      "learning_rate": 2.8821223245945747e-07,
      "logits/chosen": -1.6137434244155884,
      "logits/rejected": -1.552942156791687,
      "logps/chosen": -464.08477783203125,
      "logps/rejected": -489.72833251953125,
      "loss": 0.5231,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.016467332839966,
      "rewards/confidence": -0.37749093770980835,
      "rewards/confidence_mean_diff": 0.37749093770980835,
      "rewards/confidence_moving_diff": -0.04578464850783348,
      "rewards/margins": 0.7573314905166626,
      "rewards/mix_margin": 0.3511705994606018,
      "rewards/real_percentage": 6.599999904632568,
      "rewards/rejected": -2.773798704147339,
      "step": 310
    },
    {
      "epoch": 0.7,
      "learning_rate": 2.5440454901626487e-07,
      "logits/chosen": -1.6135238409042358,
      "logits/rejected": -1.5706686973571777,
      "logps/chosen": -462.1180725097656,
      "logps/rejected": -497.38104248046875,
      "loss": 0.5229,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.4821410179138184,
      "rewards/confidence": -0.35573774576187134,
      "rewards/confidence_mean_diff": 0.35573774576187134,
      "rewards/confidence_moving_diff": 0.010884365066885948,
      "rewards/margins": 0.6469907760620117,
      "rewards/mix_margin": 0.28311434388160706,
      "rewards/real_percentage": 6.800000190734863,
      "rewards/rejected": -3.12913179397583,
      "step": 320
    },
    {
      "epoch": 0.72,
      "learning_rate": 2.220172651615188e-07,
      "logits/chosen": -1.5926752090454102,
      "logits/rejected": -1.5634915828704834,
      "logps/chosen": -456.4151306152344,
      "logps/rejected": -458.0672912597656,
      "loss": 0.4975,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9481618404388428,
      "rewards/confidence": -0.35025110840797424,
      "rewards/confidence_mean_diff": 0.35025110840797424,
      "rewards/confidence_moving_diff": -0.0056252507492899895,
      "rewards/margins": 0.725985050201416,
      "rewards/mix_margin": 0.387787401676178,
      "rewards/real_percentage": 5.400000095367432,
      "rewards/rejected": -2.674147129058838,
      "step": 330
    },
    {
      "epoch": 0.74,
      "learning_rate": 1.9123769252690407e-07,
      "logits/chosen": -1.6937768459320068,
      "logits/rejected": -1.6783796548843384,
      "logps/chosen": -435.18768310546875,
      "logps/rejected": -442.1273498535156,
      "loss": 0.4829,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.9751695394515991,
      "rewards/confidence": -0.37928253412246704,
      "rewards/confidence_mean_diff": 0.37928253412246704,
      "rewards/confidence_moving_diff": -0.027161384001374245,
      "rewards/margins": 0.6305011510848999,
      "rewards/mix_margin": 0.3498138189315796,
      "rewards/real_percentage": 5.800000190734863,
      "rewards/rejected": -2.60567045211792,
      "step": 340
    },
    {
      "epoch": 0.76,
      "learning_rate": 1.6224384455740787e-07,
      "logits/chosen": -1.710269570350647,
      "logits/rejected": -1.652931809425354,
      "logps/chosen": -467.62591552734375,
      "logps/rejected": -491.04217529296875,
      "loss": 0.5033,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.6455416679382324,
      "rewards/confidence": -0.3581121265888214,
      "rewards/confidence_mean_diff": 0.3581121265888214,
      "rewards/confidence_moving_diff": 0.025283757597208023,
      "rewards/margins": 0.623335063457489,
      "rewards/mix_margin": 0.28707486391067505,
      "rewards/real_percentage": 7.599999904632568,
      "rewards/rejected": -3.268876552581787,
      "step": 350
    },
    {
      "epoch": 0.78,
      "learning_rate": 1.3520340697179406e-07,
      "logits/chosen": -1.6668322086334229,
      "logits/rejected": -1.6106383800506592,
      "logps/chosen": -469.1095275878906,
      "logps/rejected": -478.17449951171875,
      "loss": 0.5166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.123115062713623,
      "rewards/confidence": -0.4552002549171448,
      "rewards/confidence_mean_diff": 0.4552002549171448,
      "rewards/confidence_moving_diff": 0.0014173805247992277,
      "rewards/margins": 0.6338737607002258,
      "rewards/mix_margin": 0.3297014832496643,
      "rewards/real_percentage": 6.0,
      "rewards/rejected": -2.756988763809204,
      "step": 360
    },
    {
      "epoch": 0.81,
      "learning_rate": 1.1027276795341134e-07,
      "logits/chosen": -1.5943629741668701,
      "logits/rejected": -1.6161006689071655,
      "logps/chosen": -467.4046325683594,
      "logps/rejected": -490.90777587890625,
      "loss": 0.5108,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.3414371013641357,
      "rewards/confidence": -0.44518932700157166,
      "rewards/confidence_mean_diff": 0.44518932700157166,
      "rewards/confidence_moving_diff": 0.0043337224051356316,
      "rewards/margins": 0.6999163627624512,
      "rewards/mix_margin": 0.32578808069229126,
      "rewards/real_percentage": 6.0,
      "rewards/rejected": -3.041353464126587,
      "step": 370
    },
    {
      "epoch": 0.83,
      "learning_rate": 8.75961136802183e-08,
      "logits/chosen": -1.6814724206924438,
      "logits/rejected": -1.6028457880020142,
      "logps/chosen": -474.9368591308594,
      "logps/rejected": -513.1763916015625,
      "loss": 0.4979,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.2711257934570312,
      "rewards/confidence": -0.42974844574928284,
      "rewards/confidence_mean_diff": 0.42974844574928284,
      "rewards/confidence_moving_diff": 0.018308240920305252,
      "rewards/margins": 0.8098933100700378,
      "rewards/mix_margin": 0.3282521367073059,
      "rewards/real_percentage": 7.0,
      "rewards/rejected": -3.081019163131714,
      "step": 380
    },
    {
      "epoch": 0.85,
      "learning_rate": 6.730459442502328e-08,
      "logits/chosen": -1.6732743978500366,
      "logits/rejected": -1.6020500659942627,
      "logps/chosen": -492.012939453125,
      "logps/rejected": -516.7794189453125,
      "loss": 0.5067,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.0890817642211914,
      "rewards/confidence": -0.4012150168418884,
      "rewards/confidence_mean_diff": 0.4012150168418884,
      "rewards/confidence_moving_diff": -0.030136477202177048,
      "rewards/margins": 0.7823335528373718,
      "rewards/mix_margin": 0.3618648648262024,
      "rewards/real_percentage": 5.800000190734863,
      "rewards/rejected": -2.871415376663208,
      "step": 390
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.951556604879048e-08,
      "logits/chosen": -1.6667470932006836,
      "logits/rejected": -1.634762167930603,
      "logps/chosen": -512.33251953125,
      "logps/rejected": -515.0343017578125,
      "loss": 0.5576,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.8803839683532715,
      "rewards/confidence": -0.4185534119606018,
      "rewards/confidence_mean_diff": 0.4185534119606018,
      "rewards/confidence_moving_diff": 0.005899274256080389,
      "rewards/margins": 0.5610276460647583,
      "rewards/mix_margin": 0.29152485728263855,
      "rewards/real_percentage": 6.400000095367432,
      "rewards/rejected": -3.4414114952087402,
      "step": 400
    },
    {
      "epoch": 0.89,
      "learning_rate": 3.4331911273830784e-08,
      "logits/chosen": -1.7730276584625244,
      "logits/rejected": -1.7303050756454468,
      "logps/chosen": -459.1253967285156,
      "logps/rejected": -487.03375244140625,
      "loss": 0.5318,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.33591365814209,
      "rewards/confidence": -0.42727431654930115,
      "rewards/confidence_mean_diff": 0.42727431654930115,
      "rewards/confidence_moving_diff": 0.02859329618513584,
      "rewards/margins": 0.6499987244606018,
      "rewards/mix_margin": 0.3042239546775818,
      "rewards/real_percentage": 7.199999809265137,
      "rewards/rejected": -2.985912322998047,
      "step": 410
    },
    {
      "epoch": 0.91,
      "learning_rate": 2.184144466229154e-08,
      "logits/chosen": -1.7364966869354248,
      "logits/rejected": -1.716314673423767,
      "logps/chosen": -482.951416015625,
      "logps/rejected": -509.6161193847656,
      "loss": 0.4949,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.3884456157684326,
      "rewards/confidence": -0.4644384980201721,
      "rewards/confidence_mean_diff": 0.4644384980201721,
      "rewards/confidence_moving_diff": 0.015452265739440918,
      "rewards/margins": 0.7966889142990112,
      "rewards/mix_margin": 0.33610981702804565,
      "rewards/real_percentage": 6.400000095367432,
      "rewards/rejected": -3.1851344108581543,
      "step": 420
    },
    {
      "epoch": 0.94,
      "learning_rate": 1.2116404741244202e-08,
      "logits/chosen": -1.6397075653076172,
      "logits/rejected": -1.581413984298706,
      "logps/chosen": -482.9574279785156,
      "logps/rejected": -507.77972412109375,
      "loss": 0.4865,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.1570003032684326,
      "rewards/confidence": -0.5490925908088684,
      "rewards/confidence_mean_diff": 0.5490925908088684,
      "rewards/confidence_moving_diff": -0.04263176769018173,
      "rewards/margins": 0.6673628091812134,
      "rewards/mix_margin": 0.2867775559425354,
      "rewards/real_percentage": 5.599999904632568,
      "rewards/rejected": -2.8243632316589355,
      "step": 430
    },
    {
      "epoch": 0.96,
      "learning_rate": 5.2130362116641905e-09,
      "logits/chosen": -1.661386251449585,
      "logits/rejected": -1.655612587928772,
      "logps/chosen": -480.28643798828125,
      "logps/rejected": -493.21697998046875,
      "loss": 0.5137,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.342162609100342,
      "rewards/confidence": -0.37716978788375854,
      "rewards/confidence_mean_diff": 0.37716978788375854,
      "rewards/confidence_moving_diff": 0.004406067542731762,
      "rewards/margins": 0.6741171479225159,
      "rewards/mix_margin": 0.2875633239746094,
      "rewards/real_percentage": 6.599999904632568,
      "rewards/rejected": -3.016279935836792,
      "step": 440
    },
    {
      "epoch": 0.98,
      "learning_rate": 1.1712646575922636e-09,
      "logits/chosen": -1.7303485870361328,
      "logits/rejected": -1.7037239074707031,
      "logps/chosen": -473.63946533203125,
      "logps/rejected": -522.4127197265625,
      "loss": 0.4808,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.1520164012908936,
      "rewards/confidence": -0.3412160277366638,
      "rewards/confidence_mean_diff": 0.3412160277366638,
      "rewards/confidence_moving_diff": 0.019679775461554527,
      "rewards/margins": 1.036991000175476,
      "rewards/mix_margin": 0.3758584260940552,
      "rewards/real_percentage": 6.800000190734863,
      "rewards/rejected": -3.189007520675659,
      "step": 450
    },
    {
      "epoch": 1.0,
      "step": 459,
      "total_flos": 0.0,
      "train_loss": 0.5442263817735228,
      "train_runtime": 33688.2171,
      "train_samples_per_second": 0.873,
      "train_steps_per_second": 0.014
    }
  ],
  "logging_steps": 10,
  "max_steps": 459,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}