{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9996020692399522,
  "eval_steps": 500,
  "global_step": 1884,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 2.645502645502645e-09,
      "logits/chosen": -1.7827545404434204,
      "logits/rejected": -1.8475866317749023,
      "logps/chosen": -188.38653564453125,
      "logps/rejected": -220.2354736328125,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.6455026455026453e-08,
      "logits/chosen": -1.9714936017990112,
      "logits/rejected": -1.876778483390808,
      "logps/chosen": -288.3788146972656,
      "logps/rejected": -199.33457946777344,
      "loss": 0.6931,
      "rewards/accuracies": 0.3819444477558136,
      "rewards/chosen": -0.00010286699398420751,
      "rewards/margins": 0.00021866995666641742,
      "rewards/rejected": -0.00032153705251403153,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.2910052910052905e-08,
      "logits/chosen": -1.9301083087921143,
      "logits/rejected": -1.9124500751495361,
      "logps/chosen": -180.6164093017578,
      "logps/rejected": -186.2301483154297,
      "loss": 0.6931,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -4.3160351197002456e-05,
      "rewards/margins": -7.290135545190424e-05,
      "rewards/rejected": 2.974100425490178e-05,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.936507936507936e-08,
      "logits/chosen": -1.9063631296157837,
      "logits/rejected": -1.937107801437378,
      "logps/chosen": -190.45803833007812,
      "logps/rejected": -201.83729553222656,
      "loss": 0.6933,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.0003868662752211094,
      "rewards/margins": -0.00044071293086744845,
      "rewards/rejected": 5.3846673836233094e-05,
      "step": 30
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0582010582010581e-07,
      "logits/chosen": -1.8966213464736938,
      "logits/rejected": -1.8653703927993774,
      "logps/chosen": -212.13247680664062,
      "logps/rejected": -186.06394958496094,
      "loss": 0.693,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.00024294569448102266,
      "rewards/margins": 0.00039846505387686193,
      "rewards/rejected": -0.00015551943215541542,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3227513227513225e-07,
      "logits/chosen": -1.9822158813476562,
      "logits/rejected": -1.8613579273223877,
      "logps/chosen": -217.9025421142578,
      "logps/rejected": -174.90078735351562,
      "loss": 0.6923,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.003075198968872428,
      "rewards/margins": 0.0013211520854383707,
      "rewards/rejected": 0.001754046999849379,
      "step": 50
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5873015873015872e-07,
      "logits/chosen": -1.9449065923690796,
      "logits/rejected": -1.9192440509796143,
      "logps/chosen": -176.40371704101562,
      "logps/rejected": -172.91717529296875,
      "loss": 0.6921,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.003195743542164564,
      "rewards/margins": 0.0035919342190027237,
      "rewards/rejected": -0.0003961907350458205,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8518518518518516e-07,
      "logits/chosen": -1.9199409484863281,
      "logits/rejected": -1.8474018573760986,
      "logps/chosen": -174.99351501464844,
      "logps/rejected": -197.31735229492188,
      "loss": 0.6911,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.0023753070272505283,
      "rewards/margins": 0.006421695463359356,
      "rewards/rejected": -0.004046388436108828,
      "step": 70
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1164021164021162e-07,
      "logits/chosen": -1.8320897817611694,
      "logits/rejected": -1.844348669052124,
      "logps/chosen": -195.1573028564453,
      "logps/rejected": -195.3834228515625,
      "loss": 0.6894,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.001768267946317792,
      "rewards/margins": 0.007629372179508209,
      "rewards/rejected": -0.009397639892995358,
      "step": 80
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3809523809523806e-07,
      "logits/chosen": -1.909330129623413,
      "logits/rejected": -1.9080867767333984,
      "logps/chosen": -196.3611297607422,
      "logps/rejected": -187.08120727539062,
      "loss": 0.6879,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0239607784897089,
      "rewards/margins": 0.006291805300861597,
      "rewards/rejected": -0.030252579599618912,
      "step": 90
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.645502645502645e-07,
      "logits/chosen": -1.8753902912139893,
      "logits/rejected": -1.8588756322860718,
      "logps/chosen": -197.9770050048828,
      "logps/rejected": -210.04736328125,
      "loss": 0.6842,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.05865932255983353,
      "rewards/margins": 0.00952382292598486,
      "rewards/rejected": -0.06818314641714096,
      "step": 100
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.9100529100529097e-07,
      "logits/chosen": -1.942865014076233,
      "logits/rejected": -1.8896948099136353,
      "logps/chosen": -222.5757598876953,
      "logps/rejected": -227.91476440429688,
      "loss": 0.6796,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07471577823162079,
      "rewards/margins": 0.0264597050845623,
      "rewards/rejected": -0.1011754721403122,
      "step": 110
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1746031746031743e-07,
      "logits/chosen": -1.8088843822479248,
      "logits/rejected": -1.7518575191497803,
      "logps/chosen": -222.64022827148438,
      "logps/rejected": -209.8594512939453,
      "loss": 0.6776,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11961501836776733,
      "rewards/margins": 0.06059970706701279,
      "rewards/rejected": -0.18021473288536072,
      "step": 120
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.439153439153439e-07,
      "logits/chosen": -1.7773793935775757,
      "logits/rejected": -1.7670618295669556,
      "logps/chosen": -222.9905548095703,
      "logps/rejected": -228.2213134765625,
      "loss": 0.669,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.18275949358940125,
      "rewards/margins": 0.04632816091179848,
      "rewards/rejected": -0.22908766567707062,
      "step": 130
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.703703703703703e-07,
      "logits/chosen": -1.777341604232788,
      "logits/rejected": -1.6979541778564453,
      "logps/chosen": -227.1598358154297,
      "logps/rejected": -202.8821258544922,
      "loss": 0.6663,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.19891491532325745,
      "rewards/margins": 0.05222553759813309,
      "rewards/rejected": -0.25114041566848755,
      "step": 140
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.968253968253968e-07,
      "logits/chosen": -1.639807939529419,
      "logits/rejected": -1.5844175815582275,
      "logps/chosen": -248.316650390625,
      "logps/rejected": -248.49240112304688,
      "loss": 0.6549,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.45894140005111694,
      "rewards/margins": 0.053535498678684235,
      "rewards/rejected": -0.512476921081543,
      "step": 150
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.2328042328042324e-07,
      "logits/chosen": -1.4632054567337036,
      "logits/rejected": -1.4093120098114014,
      "logps/chosen": -265.42877197265625,
      "logps/rejected": -252.86727905273438,
      "loss": 0.6609,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6529312133789062,
      "rewards/margins": 0.09157294034957886,
      "rewards/rejected": -0.7445041537284851,
      "step": 160
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.497354497354497e-07,
      "logits/chosen": -1.4973636865615845,
      "logits/rejected": -1.3732246160507202,
      "logps/chosen": -299.1934509277344,
      "logps/rejected": -286.1786804199219,
      "loss": 0.6473,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.6386870741844177,
      "rewards/margins": 0.15349723398685455,
      "rewards/rejected": -0.7921843528747559,
      "step": 170
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.761904761904761e-07,
      "logits/chosen": -1.419722318649292,
      "logits/rejected": -1.3405072689056396,
      "logps/chosen": -262.6790466308594,
      "logps/rejected": -247.8793487548828,
      "loss": 0.6447,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6575330495834351,
      "rewards/margins": 0.12454842031002045,
      "rewards/rejected": -0.7820814847946167,
      "step": 180
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999995705919032e-07,
      "logits/chosen": -1.4580373764038086,
      "logits/rejected": -1.3449883460998535,
      "logps/chosen": -265.0930480957031,
      "logps/rejected": -266.20880126953125,
      "loss": 0.649,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6115543246269226,
      "rewards/margins": 0.11900641769170761,
      "rewards/rejected": -0.7305607795715332,
      "step": 190
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999480434051858e-07,
      "logits/chosen": -1.7100050449371338,
      "logits/rejected": -1.6133902072906494,
      "logps/chosen": -312.9882507324219,
      "logps/rejected": -298.0437927246094,
      "loss": 0.6499,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.9183656573295593,
      "rewards/margins": 0.07652232050895691,
      "rewards/rejected": -0.9948879480361938,
      "step": 200
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998106548810311e-07,
      "logits/chosen": -1.6054027080535889,
      "logits/rejected": -1.5358479022979736,
      "logps/chosen": -283.46551513671875,
      "logps/rejected": -306.21820068359375,
      "loss": 0.6371,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8442398905754089,
      "rewards/margins": 0.2072862684726715,
      "rewards/rejected": -1.0515261888504028,
      "step": 210
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995874522146975e-07,
      "logits/chosen": -1.5969936847686768,
      "logits/rejected": -1.5442743301391602,
      "logps/chosen": -294.7565002441406,
      "logps/rejected": -295.92779541015625,
      "loss": 0.6223,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.9133985638618469,
      "rewards/margins": 0.20787055790424347,
      "rewards/rejected": -1.1212691068649292,
      "step": 220
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.992785120800375e-07,
      "logits/chosen": -1.0925428867340088,
      "logits/rejected": -1.0447893142700195,
      "logps/chosen": -327.7795104980469,
      "logps/rejected": -339.69500732421875,
      "loss": 0.6349,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.108730435371399,
      "rewards/margins": 0.24315257370471954,
      "rewards/rejected": -1.3518829345703125,
      "step": 230
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.988839406031596e-07,
      "logits/chosen": -0.13352422416210175,
      "logits/rejected": 0.016487866640090942,
      "logps/chosen": -324.3548278808594,
      "logps/rejected": -332.9447326660156,
      "loss": 0.63,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.3696115016937256,
      "rewards/margins": 0.30091235041618347,
      "rewards/rejected": -1.6705236434936523,
      "step": 240
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.98403873325972e-07,
      "logits/chosen": -0.25656959414482117,
      "logits/rejected": -0.22962741553783417,
      "logps/chosen": -276.41180419921875,
      "logps/rejected": -297.1820983886719,
      "loss": 0.6098,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.8686075210571289,
      "rewards/margins": 0.2754421532154083,
      "rewards/rejected": -1.1440496444702148,
      "step": 250
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.978384751596212e-07,
      "logits/chosen": 0.7076885104179382,
      "logits/rejected": 0.7922934293746948,
      "logps/chosen": -315.2431640625,
      "logps/rejected": -349.3119812011719,
      "loss": 0.6038,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.1041444540023804,
      "rewards/margins": 0.35109204053878784,
      "rewards/rejected": -1.4552366733551025,
      "step": 260
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.971879403278432e-07,
      "logits/chosen": 1.446623682975769,
      "logits/rejected": 1.5377308130264282,
      "logps/chosen": -370.0694580078125,
      "logps/rejected": -390.1954650878906,
      "loss": 0.604,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.6548852920532227,
      "rewards/margins": 0.3384679853916168,
      "rewards/rejected": -1.9933536052703857,
      "step": 270
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.964524923002436e-07,
      "logits/chosen": 0.9782055020332336,
      "logits/rejected": 1.0214025974273682,
      "logps/chosen": -374.83990478515625,
      "logps/rejected": -407.66644287109375,
      "loss": 0.5943,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.666853666305542,
      "rewards/margins": 0.4476628303527832,
      "rewards/rejected": -2.1145167350769043,
      "step": 280
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.956323837155325e-07,
      "logits/chosen": 1.1159193515777588,
      "logits/rejected": 1.1185142993927002,
      "logps/chosen": -357.0001220703125,
      "logps/rejected": -394.30279541015625,
      "loss": 0.6107,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.6487280130386353,
      "rewards/margins": 0.3608878254890442,
      "rewards/rejected": -2.009615898132324,
      "step": 290
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.947278962947386e-07,
      "logits/chosen": 1.6991510391235352,
      "logits/rejected": 1.9209808111190796,
      "logps/chosen": -396.7066955566406,
      "logps/rejected": -431.6412658691406,
      "loss": 0.6001,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.0254971981048584,
      "rewards/margins": 0.42958149313926697,
      "rewards/rejected": -2.455078601837158,
      "step": 300
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.937393407444337e-07,
      "logits/chosen": 1.100517988204956,
      "logits/rejected": 1.2365609407424927,
      "logps/chosen": -356.17681884765625,
      "logps/rejected": -412.0787048339844,
      "loss": 0.5925,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.7612950801849365,
      "rewards/margins": 0.4073117673397064,
      "rewards/rejected": -2.168606758117676,
      "step": 310
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.926670566499992e-07,
      "logits/chosen": 1.4866538047790527,
      "logits/rejected": 1.5181796550750732,
      "logps/chosen": -370.19793701171875,
      "logps/rejected": -399.6379089355469,
      "loss": 0.5865,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.8391116857528687,
      "rewards/margins": 0.3383946418762207,
      "rewards/rejected": -2.1775062084198,
      "step": 320
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.915114123589732e-07,
      "logits/chosen": 3.2927498817443848,
      "logits/rejected": 3.4409637451171875,
      "logps/chosen": -474.81134033203125,
      "logps/rejected": -486.77264404296875,
      "loss": 0.5826,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.5577211380004883,
      "rewards/margins": 0.5131586790084839,
      "rewards/rejected": -3.0708799362182617,
      "step": 330
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.90272804854517e-07,
      "logits/chosen": 4.935120582580566,
      "logits/rejected": 5.241981029510498,
      "logps/chosen": -558.8336181640625,
      "logps/rejected": -587.92578125,
      "loss": 0.6196,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.244816303253174,
      "rewards/margins": 0.4380508065223694,
      "rewards/rejected": -3.6828665733337402,
      "step": 340
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.889516596190448e-07,
      "logits/chosen": 3.1185314655303955,
      "logits/rejected": 3.6742959022521973,
      "logps/chosen": -524.4172973632812,
      "logps/rejected": -528.1964111328125,
      "loss": 0.602,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.8538341522216797,
      "rewards/margins": 0.4846366345882416,
      "rewards/rejected": -3.338470935821533,
      "step": 350
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.875484304880629e-07,
      "logits/chosen": 1.2977662086486816,
      "logits/rejected": 1.6007190942764282,
      "logps/chosen": -468.362060546875,
      "logps/rejected": -483.639892578125,
      "loss": 0.6246,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -2.143881320953369,
      "rewards/margins": 0.36468884348869324,
      "rewards/rejected": -2.5085701942443848,
      "step": 360
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.860635994942702e-07,
      "logits/chosen": 1.5786962509155273,
      "logits/rejected": 1.9745759963989258,
      "logps/chosen": -395.7223205566406,
      "logps/rejected": -411.2752990722656,
      "loss": 0.5782,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.866349458694458,
      "rewards/margins": 0.4314461350440979,
      "rewards/rejected": -2.297795534133911,
      "step": 370
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.844976767019714e-07,
      "logits/chosen": 2.636617660522461,
      "logits/rejected": 2.620858669281006,
      "logps/chosen": -369.3752136230469,
      "logps/rejected": -416.45574951171875,
      "loss": 0.576,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.1540982723236084,
      "rewards/margins": 0.4857994019985199,
      "rewards/rejected": -2.6398978233337402,
      "step": 380
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828512000318616e-07,
      "logits/chosen": 2.6557507514953613,
      "logits/rejected": 3.367990016937256,
      "logps/chosen": -490.51708984375,
      "logps/rejected": -534.2076416015625,
      "loss": 0.5691,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.498900890350342,
      "rewards/margins": 0.7616626620292664,
      "rewards/rejected": -3.260563373565674,
      "step": 390
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.811247350762418e-07,
      "logits/chosen": 2.237056016921997,
      "logits/rejected": 2.7285072803497314,
      "logps/chosen": -381.19000244140625,
      "logps/rejected": -417.8363342285156,
      "loss": 0.5659,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.9070937633514404,
      "rewards/margins": 0.5606156587600708,
      "rewards/rejected": -2.46770977973938,
      "step": 400
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.79318874904728e-07,
      "logits/chosen": 4.107834339141846,
      "logits/rejected": 4.020180702209473,
      "logps/chosen": -422.3435974121094,
      "logps/rejected": -466.9422912597656,
      "loss": 0.5826,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -2.08524751663208,
      "rewards/margins": 0.5145748853683472,
      "rewards/rejected": -2.599822521209717,
      "step": 410
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.774342398605221e-07,
      "logits/chosen": 4.6263508796691895,
      "logits/rejected": 5.094324588775635,
      "logps/chosen": -447.66790771484375,
      "logps/rejected": -472.98541259765625,
      "loss": 0.5864,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.5026144981384277,
      "rewards/margins": 0.5022978782653809,
      "rewards/rejected": -3.0049126148223877,
      "step": 420
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.754714773473134e-07,
      "logits/chosen": 3.137293577194214,
      "logits/rejected": 3.4239907264709473,
      "logps/chosen": -383.85272216796875,
      "logps/rejected": -439.3240661621094,
      "loss": 0.5614,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.8029075860977173,
      "rewards/margins": 0.6045055389404297,
      "rewards/rejected": -2.4074130058288574,
      "step": 430
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.734312616068851e-07,
      "logits/chosen": 3.090550184249878,
      "logits/rejected": 3.005370855331421,
      "logps/chosen": -393.83636474609375,
      "logps/rejected": -456.08917236328125,
      "loss": 0.5636,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2565479278564453,
      "rewards/margins": 0.5205822587013245,
      "rewards/rejected": -2.777130365371704,
      "step": 440
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.713142934875005e-07,
      "logits/chosen": 3.673936128616333,
      "logits/rejected": 4.1635026931762695,
      "logps/chosen": -492.8273010253906,
      "logps/rejected": -526.05810546875,
      "loss": 0.5837,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.6836562156677246,
      "rewards/margins": 0.5476377010345459,
      "rewards/rejected": -3.2312939167022705,
      "step": 450
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.6912130020314996e-07,
      "logits/chosen": 3.0116238594055176,
      "logits/rejected": 3.2378668785095215,
      "logps/chosen": -389.37811279296875,
      "logps/rejected": -445.2620544433594,
      "loss": 0.5657,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -2.161726951599121,
      "rewards/margins": 0.5274554491043091,
      "rewards/rejected": -2.6891825199127197,
      "step": 460
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.668530350837408e-07,
      "logits/chosen": 2.4322941303253174,
      "logits/rejected": 2.488206148147583,
      "logps/chosen": -389.4209899902344,
      "logps/rejected": -435.225341796875,
      "loss": 0.5843,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.8818559646606445,
      "rewards/margins": 0.40773385763168335,
      "rewards/rejected": -2.2895896434783936,
      "step": 470
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.64510277316316e-07,
      "logits/chosen": 2.517045736312866,
      "logits/rejected": 2.917738676071167,
      "logps/chosen": -429.2723693847656,
      "logps/rejected": -474.98681640625,
      "loss": 0.568,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.0465638637542725,
      "rewards/margins": 0.6840267777442932,
      "rewards/rejected": -2.730590581893921,
      "step": 480
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6209383167739015e-07,
      "logits/chosen": 2.8990161418914795,
      "logits/rejected": 3.3626670837402344,
      "logps/chosen": -449.22235107421875,
      "logps/rejected": -482.134033203125,
      "loss": 0.5814,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -2.392901659011841,
      "rewards/margins": 0.5466801524162292,
      "rewards/rejected": -2.939582109451294,
      "step": 490
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5960452825649526e-07,
      "logits/chosen": 2.796527147293091,
      "logits/rejected": 2.8022494316101074,
      "logps/chosen": -416.53692626953125,
      "logps/rejected": -469.519287109375,
      "loss": 0.5481,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.1346237659454346,
      "rewards/margins": 0.5421117544174194,
      "rewards/rejected": -2.6767354011535645,
      "step": 500
    },
    {
      "epoch": 0.27,
      "eval_logits/chosen": 4.160411834716797,
      "eval_logits/rejected": 3.967071294784546,
      "eval_logps/chosen": -386.0565185546875,
      "eval_logps/rejected": -439.2520751953125,
      "eval_loss": 0.6089237928390503,
      "eval_rewards/accuracies": 0.6705063581466675,
      "eval_rewards/chosen": -2.6821529865264893,
      "eval_rewards/margins": 0.44139912724494934,
      "eval_rewards/rejected": -3.123551845550537,
      "eval_runtime": 2833.1592,
      "eval_samples_per_second": 12.369,
      "eval_steps_per_second": 0.387,
      "step": 500
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.570432221710314e-07,
      "logits/chosen": 2.99139666557312,
      "logits/rejected": 2.9831995964050293,
      "logps/chosen": -414.53643798828125,
      "logps/rejected": -447.3922424316406,
      "loss": 0.5896,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.9070537090301514,
      "rewards/margins": 0.6989824175834656,
      "rewards/rejected": -2.6060359477996826,
      "step": 510
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.5441079327251927e-07,
      "logits/chosen": 2.8209099769592285,
      "logits/rejected": 3.297544002532959,
      "logps/chosen": -415.64990234375,
      "logps/rejected": -483.9609375,
      "loss": 0.5711,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -2.1026933193206787,
      "rewards/margins": 0.5386781096458435,
      "rewards/rejected": -2.641371726989746,
      "step": 520
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.5170814584435644e-07,
      "logits/chosen": 3.1211986541748047,
      "logits/rejected": 3.634293794631958,
      "logps/chosen": -445.901611328125,
      "logps/rejected": -477.486083984375,
      "loss": 0.5723,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.098557949066162,
      "rewards/margins": 0.7294325828552246,
      "rewards/rejected": -2.8279905319213867,
      "step": 530
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4893620829118124e-07,
      "logits/chosen": 4.876758575439453,
      "logits/rejected": 4.832733154296875,
      "logps/chosen": -390.5097351074219,
      "logps/rejected": -458.13433837890625,
      "loss": 0.5553,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.3329362869262695,
      "rewards/margins": 0.6324361562728882,
      "rewards/rejected": -2.965372323989868,
      "step": 540
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.460959328199497e-07,
      "logits/chosen": 5.745593070983887,
      "logits/rejected": 6.544689178466797,
      "logps/chosen": -479.26312255859375,
      "logps/rejected": -547.4561767578125,
      "loss": 0.5592,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -2.804473876953125,
      "rewards/margins": 0.6489624977111816,
      "rewards/rejected": -3.4534363746643066,
      "step": 550
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.4318829511283707e-07,
      "logits/chosen": 5.760529518127441,
      "logits/rejected": 5.908038139343262,
      "logps/chosen": -500.1220703125,
      "logps/rejected": -579.2546997070312,
      "loss": 0.5787,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -3.121411085128784,
      "rewards/margins": 0.5803627967834473,
      "rewards/rejected": -3.7017738819122314,
      "step": 560
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.40214293992074e-07,
      "logits/chosen": 4.968967914581299,
      "logits/rejected": 5.733763694763184,
      "logps/chosen": -449.43182373046875,
      "logps/rejected": -481.9095153808594,
      "loss": 0.5705,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.2817494869232178,
      "rewards/margins": 0.5884373188018799,
      "rewards/rejected": -2.8701870441436768,
      "step": 570
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3717495107683516e-07,
      "logits/chosen": 6.087460041046143,
      "logits/rejected": 6.749306678771973,
      "logps/chosen": -554.03173828125,
      "logps/rejected": -600.6712646484375,
      "loss": 0.5706,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -3.579016923904419,
      "rewards/margins": 0.5537681579589844,
      "rewards/rejected": -4.132784843444824,
      "step": 580
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.340713104322953e-07,
      "logits/chosen": 5.846838474273682,
      "logits/rejected": 6.166060447692871,
      "logps/chosen": -547.8532104492188,
      "logps/rejected": -635.3438110351562,
      "loss": 0.5318,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.4989750385284424,
      "rewards/margins": 0.7716847658157349,
      "rewards/rejected": -4.270659446716309,
      "step": 590
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.3090443821097566e-07,
      "logits/chosen": 4.9042229652404785,
      "logits/rejected": 5.197253704071045,
      "logps/chosen": -483.651123046875,
      "logps/rejected": -557.0797119140625,
      "loss": 0.5436,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.6425604820251465,
      "rewards/margins": 0.6980206370353699,
      "rewards/rejected": -3.340581178665161,
      "step": 600
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.276754222865029e-07,
      "logits/chosen": 4.094830513000488,
      "logits/rejected": 4.941472053527832,
      "logps/chosen": -461.6971130371094,
      "logps/rejected": -490.43377685546875,
      "loss": 0.5627,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -2.5023880004882812,
      "rewards/margins": 0.6450821757316589,
      "rewards/rejected": -3.147469997406006,
      "step": 610
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2438537187990565e-07,
      "logits/chosen": 4.136041641235352,
      "logits/rejected": 4.636130332946777,
      "logps/chosen": -462.34136962890625,
      "logps/rejected": -513.6154174804688,
      "loss": 0.5457,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -2.5274147987365723,
      "rewards/margins": 0.8207530975341797,
      "rewards/rejected": -3.348167896270752,
      "step": 620
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.210354171785795e-07,
      "logits/chosen": 4.823207855224609,
      "logits/rejected": 5.325142860412598,
      "logps/chosen": -488.6162109375,
      "logps/rejected": -548.1345825195312,
      "loss": 0.5523,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.699159622192383,
      "rewards/margins": 0.712811291217804,
      "rewards/rejected": -3.4119713306427,
      "step": 630
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1762670894804775e-07,
      "logits/chosen": 4.793757915496826,
      "logits/rejected": 5.079339027404785,
      "logps/chosen": -517.7249755859375,
      "logps/rejected": -555.1099243164062,
      "loss": 0.5609,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -3.0854828357696533,
      "rewards/margins": 0.5394307374954224,
      "rewards/rejected": -3.624913454055786,
      "step": 640
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1416041813665493e-07,
      "logits/chosen": 4.005798816680908,
      "logits/rejected": 4.380357265472412,
      "logps/chosen": -465.93701171875,
      "logps/rejected": -518.0279541015625,
      "loss": 0.5483,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.6373207569122314,
      "rewards/margins": 0.5303686857223511,
      "rewards/rejected": -3.167689323425293,
      "step": 650
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.1063773547332584e-07,
      "logits/chosen": 4.460536956787109,
      "logits/rejected": 4.873723030090332,
      "logps/chosen": -462.9532775878906,
      "logps/rejected": -519.1263427734375,
      "loss": 0.5651,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -2.6806063652038574,
      "rewards/margins": 0.6171140670776367,
      "rewards/rejected": -3.2977206707000732,
      "step": 660
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0705987105853077e-07,
      "logits/chosen": 4.531912803649902,
      "logits/rejected": 4.875602722167969,
      "logps/chosen": -430.66015625,
      "logps/rejected": -490.43505859375,
      "loss": 0.5664,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.494072437286377,
      "rewards/margins": 0.5989924669265747,
      "rewards/rejected": -3.0930652618408203,
      "step": 670
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.034280539485952e-07,
      "logits/chosen": 3.7882587909698486,
      "logits/rejected": 4.701764106750488,
      "logps/chosen": -472.9560546875,
      "logps/rejected": -545.2316284179688,
      "loss": 0.5388,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.2864925861358643,
      "rewards/margins": 0.8441749811172485,
      "rewards/rejected": -3.130667209625244,
      "step": 680
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.997435317334988e-07,
      "logits/chosen": 5.6066789627075195,
      "logits/rejected": 6.343667507171631,
      "logps/chosen": -546.0533447265625,
      "logps/rejected": -598.7782592773438,
      "loss": 0.5491,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -3.1853551864624023,
      "rewards/margins": 0.7388944625854492,
      "rewards/rejected": -3.9242501258850098,
      "step": 690
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.960075701083074e-07,
      "logits/chosen": 6.01334285736084,
      "logits/rejected": 6.114634037017822,
      "logps/chosen": -553.4805908203125,
      "logps/rejected": -612.23095703125,
      "loss": 0.5694,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -3.704514980316162,
      "rewards/margins": 0.5731467008590698,
      "rewards/rejected": -4.27766227722168,
      "step": 700
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.92221452438385e-07,
      "logits/chosen": 5.397694110870361,
      "logits/rejected": 5.5348615646362305,
      "logps/chosen": -530.5993041992188,
      "logps/rejected": -563.1759033203125,
      "loss": 0.5445,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -3.2607131004333496,
      "rewards/margins": 0.5572507977485657,
      "rewards/rejected": -3.817964553833008,
      "step": 710
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.8838647931853684e-07,
      "logits/chosen": 5.427793025970459,
      "logits/rejected": 5.946757793426514,
      "logps/chosen": -589.3401489257812,
      "logps/rejected": -668.6383666992188,
      "loss": 0.5744,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -3.8846659660339355,
      "rewards/margins": 0.7481729388237,
      "rewards/rejected": -4.632839679718018,
      "step": 720
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.845039681262332e-07,
      "logits/chosen": 5.047539710998535,
      "logits/rejected": 5.657217025756836,
      "logps/chosen": -554.4708862304688,
      "logps/rejected": -616.8558349609375,
      "loss": 0.5279,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -3.465214967727661,
      "rewards/margins": 0.6963861584663391,
      "rewards/rejected": -4.1616010665893555,
      "step": 730
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.805752525690681e-07,
      "logits/chosen": 5.46529483795166,
      "logits/rejected": 5.889483451843262,
      "logps/chosen": -564.59228515625,
      "logps/rejected": -660.7513427734375,
      "loss": 0.5431,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.814227342605591,
      "rewards/margins": 0.7818762063980103,
      "rewards/rejected": -4.596103191375732,
      "step": 740
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7660168222660824e-07,
      "logits/chosen": 5.152674198150635,
      "logits/rejected": 5.510188102722168,
      "logps/chosen": -587.24560546875,
      "logps/rejected": -626.5244140625,
      "loss": 0.5788,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.4678547382354736,
      "rewards/margins": 0.5621030926704407,
      "rewards/rejected": -4.0299577713012695,
      "step": 750
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.725846220867901e-07,
      "logits/chosen": 4.9715166091918945,
      "logits/rejected": 5.927099227905273,
      "logps/chosen": -540.8013916015625,
      "logps/rejected": -573.8363037109375,
      "loss": 0.5665,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -3.4430599212646484,
      "rewards/margins": 0.5011590123176575,
      "rewards/rejected": -3.944218873977661,
      "step": 760
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6852545207702393e-07,
      "logits/chosen": 5.292831897735596,
      "logits/rejected": 6.226684093475342,
      "logps/chosen": -585.9905395507812,
      "logps/rejected": -621.8675537109375,
      "loss": 0.5763,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.4966049194335938,
      "rewards/margins": 0.7289837598800659,
      "rewards/rejected": -4.225588798522949,
      "step": 770
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6442556659016475e-07,
      "logits/chosen": 5.347090244293213,
      "logits/rejected": 6.540925025939941,
      "logps/chosen": -568.36572265625,
      "logps/rejected": -611.6026611328125,
      "loss": 0.5235,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.558441162109375,
      "rewards/margins": 0.7847374081611633,
      "rewards/rejected": -4.343178749084473,
      "step": 780
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.602863740055161e-07,
      "logits/chosen": 5.637946128845215,
      "logits/rejected": 6.208116054534912,
      "logps/chosen": -585.2955322265625,
      "logps/rejected": -676.1448974609375,
      "loss": 0.5476,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.8783912658691406,
      "rewards/margins": 0.8531737327575684,
      "rewards/rejected": -4.731564998626709,
      "step": 790
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5610929620502747e-07,
      "logits/chosen": 4.267381191253662,
      "logits/rejected": 5.2271528244018555,
      "logps/chosen": -578.3743286132812,
      "logps/rejected": -665.9542846679688,
      "loss": 0.5381,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.779019594192505,
      "rewards/margins": 0.8331762552261353,
      "rewards/rejected": -4.61219596862793,
      "step": 800
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5189576808485404e-07,
      "logits/chosen": 4.498036861419678,
      "logits/rejected": 5.546736717224121,
      "logps/chosen": -622.6543579101562,
      "logps/rejected": -662.39990234375,
      "loss": 0.5631,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.9034676551818848,
      "rewards/margins": 0.7405894994735718,
      "rewards/rejected": -4.644057273864746,
      "step": 810
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.476472370624464e-07,
      "logits/chosen": 5.130145072937012,
      "logits/rejected": 5.417891502380371,
      "logps/chosen": -601.1119384765625,
      "logps/rejected": -665.1697387695312,
      "loss": 0.5375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -4.171017646789551,
      "rewards/margins": 0.6281455159187317,
      "rewards/rejected": -4.799162864685059,
      "step": 820
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.43365162579338e-07,
      "logits/chosen": 5.479987144470215,
      "logits/rejected": 5.379258155822754,
      "logps/chosen": -553.6651611328125,
      "logps/rejected": -656.02880859375,
      "loss": 0.5448,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.905271053314209,
      "rewards/margins": 0.8390300869941711,
      "rewards/rejected": -4.744300842285156,
      "step": 830
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.390510155998023e-07,
      "logits/chosen": 5.235553741455078,
      "logits/rejected": 5.778375148773193,
      "logps/chosen": -578.1744384765625,
      "logps/rejected": -612.1620483398438,
      "loss": 0.5477,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.762741804122925,
      "rewards/margins": 0.507239043712616,
      "rewards/rejected": -4.269980430603027,
      "step": 840
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.347062781055526e-07,
      "logits/chosen": 5.177485942840576,
      "logits/rejected": 5.3884077072143555,
      "logps/chosen": -517.0951538085938,
      "logps/rejected": -611.8797607421875,
      "loss": 0.5674,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.408946990966797,
      "rewards/margins": 0.6858997344970703,
      "rewards/rejected": -4.094846725463867,
      "step": 850
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.303324425866559e-07,
      "logits/chosen": 4.779083728790283,
      "logits/rejected": 4.99451208114624,
      "logps/chosen": -534.8001708984375,
      "logps/rejected": -588.9842529296875,
      "loss": 0.5692,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -3.043729543685913,
      "rewards/margins": 0.7375171184539795,
      "rewards/rejected": -3.7812466621398926,
      "step": 860
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2593101152883795e-07,
      "logits/chosen": 4.893560409545898,
      "logits/rejected": 4.912612438201904,
      "logps/chosen": -477.8765563964844,
      "logps/rejected": -577.2595825195312,
      "loss": 0.5559,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.9390878677368164,
      "rewards/margins": 0.735144853591919,
      "rewards/rejected": -3.6742329597473145,
      "step": 870
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.21503496897354e-07,
      "logits/chosen": 4.999721527099609,
      "logits/rejected": 5.3239426612854,
      "logps/chosen": -539.9871215820312,
      "logps/rejected": -568.0904541015625,
      "loss": 0.5742,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -3.2623775005340576,
      "rewards/margins": 0.4343135356903076,
      "rewards/rejected": -3.6966910362243652,
      "step": 880
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.170514196176037e-07,
      "logits/chosen": 4.89313268661499,
      "logits/rejected": 4.976138114929199,
      "logps/chosen": -531.0595703125,
      "logps/rejected": -615.2716064453125,
      "loss": 0.5382,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.470829486846924,
      "rewards/margins": 0.6645902395248413,
      "rewards/rejected": -4.1354193687438965,
      "step": 890
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.125763090526674e-07,
      "logits/chosen": 4.8970746994018555,
      "logits/rejected": 5.335020542144775,
      "logps/chosen": -570.1527709960938,
      "logps/rejected": -649.9376831054688,
      "loss": 0.5099,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.5448403358459473,
      "rewards/margins": 0.861894965171814,
      "rewards/rejected": -4.406734943389893,
      "step": 900
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.080797024779447e-07,
      "logits/chosen": 4.570401191711426,
      "logits/rejected": 5.245053291320801,
      "logps/chosen": -538.8987426757812,
      "logps/rejected": -598.7467651367188,
      "loss": 0.529,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.5106894969940186,
      "rewards/margins": 0.7811245918273926,
      "rewards/rejected": -4.29181432723999,
      "step": 910
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.035631445530743e-07,
      "logits/chosen": 5.412999629974365,
      "logits/rejected": 5.755091190338135,
      "logps/chosen": -584.6710205078125,
      "logps/rejected": -669.8858642578125,
      "loss": 0.5345,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.7473666667938232,
      "rewards/margins": 0.8245238065719604,
      "rewards/rejected": -4.571890830993652,
      "step": 920
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.9902818679131775e-07,
      "logits/chosen": 5.077193260192871,
      "logits/rejected": 5.424740791320801,
      "logps/chosen": -592.6712646484375,
      "logps/rejected": -663.4064331054688,
      "loss": 0.5308,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -4.113033294677734,
      "rewards/margins": 0.7320185899734497,
      "rewards/rejected": -4.845052242279053,
      "step": 930
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.944763870265886e-07,
      "logits/chosen": 4.157199859619141,
      "logits/rejected": 5.055262565612793,
      "logps/chosen": -593.670166015625,
      "logps/rejected": -670.92578125,
      "loss": 0.5311,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.861306667327881,
      "rewards/margins": 0.8220443725585938,
      "rewards/rejected": -4.683350563049316,
      "step": 940
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.899093088783105e-07,
      "logits/chosen": 4.214892864227295,
      "logits/rejected": 4.872607707977295,
      "logps/chosen": -627.0443115234375,
      "logps/rejected": -696.4767456054688,
      "loss": 0.5336,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.9733481407165527,
      "rewards/margins": 0.7376152276992798,
      "rewards/rejected": -4.710963726043701,
      "step": 950
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8532852121428733e-07,
      "logits/chosen": 4.1724419593811035,
      "logits/rejected": 4.470523834228516,
      "logps/chosen": -547.5701904296875,
      "logps/rejected": -630.979736328125,
      "loss": 0.5351,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.5739071369171143,
      "rewards/margins": 0.9093371629714966,
      "rewards/rejected": -4.4832444190979,
      "step": 960
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.807355976117716e-07,
      "logits/chosen": 3.997878313064575,
      "logits/rejected": 4.5441575050354,
      "logps/chosen": -573.764892578125,
      "logps/rejected": -626.8602294921875,
      "loss": 0.5637,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -3.4168217182159424,
      "rewards/margins": 0.7197359800338745,
      "rewards/rejected": -4.1365580558776855,
      "step": 970
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.761321158169134e-07,
      "logits/chosen": 3.821613311767578,
      "logits/rejected": 4.272428512573242,
      "logps/chosen": -539.5350341796875,
      "logps/rejected": -624.6610717773438,
      "loss": 0.5437,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.425632953643799,
      "rewards/margins": 0.7200672030448914,
      "rewards/rejected": -4.145700454711914,
      "step": 980
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.715196572027789e-07,
      "logits/chosen": 3.830516815185547,
      "logits/rejected": 3.83209228515625,
      "logps/chosen": -553.0765380859375,
      "logps/rejected": -614.8712768554688,
      "loss": 0.5591,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.4271316528320312,
      "rewards/margins": 0.6863579750061035,
      "rewards/rejected": -4.113489627838135,
      "step": 990
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6689980622612204e-07,
      "logits/chosen": 4.027700424194336,
      "logits/rejected": 4.4086809158325195,
      "logps/chosen": -504.0587463378906,
      "logps/rejected": -589.3375854492188,
      "loss": 0.5519,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -3.1788105964660645,
      "rewards/margins": 0.6995212435722351,
      "rewards/rejected": -3.8783316612243652,
      "step": 1000
    },
    {
      "epoch": 0.53,
      "eval_logits/chosen": 5.348609447479248,
      "eval_logits/rejected": 5.197441101074219,
      "eval_logps/chosen": -543.0738525390625,
      "eval_logps/rejected": -602.8671264648438,
      "eval_loss": 0.5866538882255554,
      "eval_rewards/accuracies": 0.6894388794898987,
      "eval_rewards/chosen": -4.252326488494873,
      "eval_rewards/margins": 0.5073760151863098,
      "eval_rewards/rejected": -4.759702205657959,
      "eval_runtime": 2829.1466,
      "eval_samples_per_second": 12.387,
      "eval_steps_per_second": 0.387,
      "step": 1000
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.622741498830969e-07,
      "logits/chosen": 4.356446266174316,
      "logits/rejected": 4.390655517578125,
      "logps/chosen": -531.9536743164062,
      "logps/rejected": -595.4317626953125,
      "loss": 0.5685,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.2892327308654785,
      "rewards/margins": 0.6568211913108826,
      "rewards/rejected": -3.9460537433624268,
      "step": 1010
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5764427716409815e-07,
      "logits/chosen": 3.622380495071411,
      "logits/rejected": 4.40954065322876,
      "logps/chosen": -515.4381103515625,
      "logps/rejected": -565.3131103515625,
      "loss": 0.5428,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -3.0056488513946533,
      "rewards/margins": 0.683438777923584,
      "rewards/rejected": -3.6890876293182373,
      "step": 1020
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5301177850791616e-07,
      "logits/chosen": 3.9004855155944824,
      "logits/rejected": 4.826648712158203,
      "logps/chosen": -568.6829833984375,
      "logps/rejected": -630.1458740234375,
      "loss": 0.5394,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.4316725730895996,
      "rewards/margins": 0.825964093208313,
      "rewards/rejected": -4.257637023925781,
      "step": 1030
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4837824525539477e-07,
      "logits/chosen": 3.6745972633361816,
      "logits/rejected": 4.580860614776611,
      "logps/chosen": -539.4935302734375,
      "logps/rejected": -591.1002197265625,
      "loss": 0.5549,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -3.340437650680542,
      "rewards/margins": 0.6588428020477295,
      "rewards/rejected": -3.9992804527282715,
      "step": 1040
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4374526910277886e-07,
      "logits/chosen": 4.059652805328369,
      "logits/rejected": 4.569832801818848,
      "logps/chosen": -524.096435546875,
      "logps/rejected": -599.4136962890625,
      "loss": 0.566,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.2275619506835938,
      "rewards/margins": 0.767623782157898,
      "rewards/rejected": -3.9951858520507812,
      "step": 1050
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.391144415549403e-07,
      "logits/chosen": 4.6053290367126465,
      "logits/rejected": 4.822667121887207,
      "logps/chosen": -529.7521362304688,
      "logps/rejected": -581.6422729492188,
      "loss": 0.5563,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -3.478646755218506,
      "rewards/margins": 0.4808022081851959,
      "rewards/rejected": -3.959449052810669,
      "step": 1060
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3448735337866919e-07,
      "logits/chosen": 4.680068016052246,
      "logits/rejected": 5.09098482131958,
      "logps/chosen": -486.0177307128906,
      "logps/rejected": -539.49658203125,
      "loss": 0.5976,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.101942539215088,
      "rewards/margins": 0.5219632983207703,
      "rewards/rejected": -3.623905897140503,
      "step": 1070
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.2986559405621886e-07,
      "logits/chosen": 4.115077495574951,
      "logits/rejected": 4.646478652954102,
      "logps/chosen": -508.98480224609375,
      "logps/rejected": -547.0084838867188,
      "loss": 0.5705,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.912672281265259,
      "rewards/margins": 0.5327907204627991,
      "rewards/rejected": -3.445463180541992,
      "step": 1080
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2525075123929213e-07,
      "logits/chosen": 4.412012100219727,
      "logits/rejected": 4.801966190338135,
      "logps/chosen": -494.9298400878906,
      "logps/rejected": -553.3126220703125,
      "loss": 0.53,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.0377612113952637,
      "rewards/margins": 0.6897190809249878,
      "rewards/rejected": -3.72748064994812,
      "step": 1090
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.206444102036565e-07,
      "logits/chosen": 4.899998664855957,
      "logits/rejected": 5.011576175689697,
      "logps/chosen": -533.6759643554688,
      "logps/rejected": -604.164306640625,
      "loss": 0.5622,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.346374034881592,
      "rewards/margins": 0.5992105603218079,
      "rewards/rejected": -3.945584774017334,
      "step": 1100
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.160481533045751e-07,
      "logits/chosen": 4.868978977203369,
      "logits/rejected": 5.403256416320801,
      "logps/chosen": -569.0045776367188,
      "logps/rejected": -625.4342041015625,
      "loss": 0.547,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.556847333908081,
      "rewards/margins": 0.7111159563064575,
      "rewards/rejected": -4.267963409423828,
      "step": 1110
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1146355943324148e-07,
      "logits/chosen": 5.191876411437988,
      "logits/rejected": 5.507510185241699,
      "logps/chosen": -557.5202026367188,
      "logps/rejected": -637.4208374023438,
      "loss": 0.5456,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.72900652885437,
      "rewards/margins": 0.7132843732833862,
      "rewards/rejected": -4.442290782928467,
      "step": 1120
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0689220347440374e-07,
      "logits/chosen": 4.4830732345581055,
      "logits/rejected": 5.013645172119141,
      "logps/chosen": -587.7266235351562,
      "logps/rejected": -625.2584228515625,
      "loss": 0.5546,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.556133985519409,
      "rewards/margins": 0.6282913684844971,
      "rewards/rejected": -4.184426307678223,
      "step": 1130
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0233565576536564e-07,
      "logits/chosen": 4.331673622131348,
      "logits/rejected": 4.588160037994385,
      "logps/chosen": -579.3844604492188,
      "logps/rejected": -643.15380859375,
      "loss": 0.5457,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -3.580660581588745,
      "rewards/margins": 0.58534175157547,
      "rewards/rejected": -4.16600227355957,
      "step": 1140
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.97795481556549e-07,
      "logits/chosen": 3.8659698963165283,
      "logits/rejected": 4.494038105010986,
      "logps/chosen": -526.2404174804688,
      "logps/rejected": -560.0923461914062,
      "loss": 0.5521,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.2603347301483154,
      "rewards/margins": 0.6364775896072388,
      "rewards/rejected": -3.8968119621276855,
      "step": 1150
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9327324047380422e-07,
      "logits/chosen": 3.7696309089660645,
      "logits/rejected": 4.594429016113281,
      "logps/chosen": -504.9520568847656,
      "logps/rejected": -586.5645751953125,
      "loss": 0.5119,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.0566916465759277,
      "rewards/margins": 0.8342288732528687,
      "rewards/rejected": -3.8909201622009277,
      "step": 1160
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.887704859826528e-07,
      "logits/chosen": 4.6678924560546875,
      "logits/rejected": 5.033539772033691,
      "logps/chosen": -579.7088012695312,
      "logps/rejected": -635.5621948242188,
      "loss": 0.5359,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -3.5427486896514893,
      "rewards/margins": 0.6539695262908936,
      "rewards/rejected": -4.196718692779541,
      "step": 1170
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8428876485464572e-07,
      "logits/chosen": 4.120070934295654,
      "logits/rejected": 4.802687644958496,
      "logps/chosen": -546.7890014648438,
      "logps/rejected": -595.0975341796875,
      "loss": 0.5792,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -3.6790192127227783,
      "rewards/margins": 0.6043803095817566,
      "rewards/rejected": -4.28339958190918,
      "step": 1180
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.798296166360216e-07,
      "logits/chosen": 4.212487697601318,
      "logits/rejected": 5.026057243347168,
      "logps/chosen": -570.1341552734375,
      "logps/rejected": -631.1887817382812,
      "loss": 0.554,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.4938530921936035,
      "rewards/margins": 0.8111255764961243,
      "rewards/rejected": -4.304978370666504,
      "step": 1190
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7539457311884675e-07,
      "logits/chosen": 4.184502601623535,
      "logits/rejected": 4.767962455749512,
      "logps/chosen": -532.1156005859375,
      "logps/rejected": -596.5296630859375,
      "loss": 0.5289,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.360400438308716,
      "rewards/margins": 0.7973008155822754,
      "rewards/rejected": -4.157701015472412,
      "step": 1200
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7098515781481883e-07,
      "logits/chosen": 5.043318748474121,
      "logits/rejected": 5.477365970611572,
      "logps/chosen": -554.041015625,
      "logps/rejected": -602.6400146484375,
      "loss": 0.5381,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.55816388130188,
      "rewards/margins": 0.6002733111381531,
      "rewards/rejected": -4.1584367752075195,
      "step": 1210
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6660288543191568e-07,
      "logits/chosen": 4.342132568359375,
      "logits/rejected": 5.391766548156738,
      "logps/chosen": -545.3358764648438,
      "logps/rejected": -593.21044921875,
      "loss": 0.5273,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.2604167461395264,
      "rewards/margins": 0.7034675478935242,
      "rewards/rejected": -3.9638848304748535,
      "step": 1220
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6224926135406693e-07,
      "logits/chosen": 4.523472785949707,
      "logits/rejected": 4.9561238288879395,
      "logps/chosen": -541.8357543945312,
      "logps/rejected": -617.3576049804688,
      "loss": 0.5279,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.246079206466675,
      "rewards/margins": 0.7863640785217285,
      "rewards/rejected": -4.032443046569824,
      "step": 1230
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.579257811240298e-07,
      "logits/chosen": 4.174516201019287,
      "logits/rejected": 4.948155403137207,
      "logps/chosen": -566.966796875,
      "logps/rejected": -629.2576293945312,
      "loss": 0.5383,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -3.5496108531951904,
      "rewards/margins": 0.6297346949577332,
      "rewards/rejected": -4.179345607757568,
      "step": 1240
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5363392992964523e-07,
      "logits/chosen": 4.812657833099365,
      "logits/rejected": 5.1759843826293945,
      "logps/chosen": -545.2940673828125,
      "logps/rejected": -610.6871948242188,
      "loss": 0.5599,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -3.605778217315674,
      "rewards/margins": 0.5686632394790649,
      "rewards/rejected": -4.174441337585449,
      "step": 1250
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4937518209365108e-07,
      "logits/chosen": 4.399840354919434,
      "logits/rejected": 5.024414539337158,
      "logps/chosen": -565.787353515625,
      "logps/rejected": -621.1328125,
      "loss": 0.5422,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.319702625274658,
      "rewards/margins": 0.7489235997200012,
      "rewards/rejected": -4.0686259269714355,
      "step": 1260
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4515100056722708e-07,
      "logits/chosen": 4.322859287261963,
      "logits/rejected": 4.647487163543701,
      "logps/chosen": -473.44915771484375,
      "logps/rejected": -556.26806640625,
      "loss": 0.5335,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.9221551418304443,
      "rewards/margins": 0.8288917541503906,
      "rewards/rejected": -3.7510476112365723,
      "step": 1270
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4096283642744716e-07,
      "logits/chosen": 4.445618152618408,
      "logits/rejected": 5.310964107513428,
      "logps/chosen": -529.9594116210938,
      "logps/rejected": -583.6397094726562,
      "loss": 0.5489,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.098684310913086,
      "rewards/margins": 0.7321470975875854,
      "rewards/rejected": -3.8308310508728027,
      "step": 1280
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3681212837880977e-07,
      "logits/chosen": 4.398858070373535,
      "logits/rejected": 4.8625688552856445,
      "logps/chosen": -509.6946716308594,
      "logps/rejected": -594.6348876953125,
      "loss": 0.5397,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -2.9974119663238525,
      "rewards/margins": 0.8787434697151184,
      "rewards/rejected": -3.876155376434326,
      "step": 1290
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3270030225901908e-07,
      "logits/chosen": 4.207759857177734,
      "logits/rejected": 5.357243537902832,
      "logps/chosen": -540.0505981445312,
      "logps/rejected": -591.9456787109375,
      "loss": 0.5405,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.9670040607452393,
      "rewards/margins": 0.9191484451293945,
      "rewards/rejected": -3.8861522674560547,
      "step": 1300
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2862877054918572e-07,
      "logits/chosen": 3.810641050338745,
      "logits/rejected": 4.167244911193848,
      "logps/chosen": -455.54962158203125,
      "logps/rejected": -525.954833984375,
      "loss": 0.558,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.536816358566284,
      "rewards/margins": 0.7640112638473511,
      "rewards/rejected": -3.300827741622925,
      "step": 1310
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2459893188861613e-07,
      "logits/chosen": 3.5092411041259766,
      "logits/rejected": 4.095297813415527,
      "logps/chosen": -398.01702880859375,
      "logps/rejected": -486.40106201171875,
      "loss": 0.5194,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.352292060852051,
      "rewards/margins": 0.8707995414733887,
      "rewards/rejected": -3.2230911254882812,
      "step": 1320
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.206121705943558e-07,
      "logits/chosen": 3.8648457527160645,
      "logits/rejected": 4.79974365234375,
      "logps/chosen": -450.3089904785156,
      "logps/rejected": -489.1319274902344,
      "loss": 0.5501,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.3793253898620605,
      "rewards/margins": 0.6881033778190613,
      "rewards/rejected": -3.0674290657043457,
      "step": 1330
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1666985618565422e-07,
      "logits/chosen": 4.9529547691345215,
      "logits/rejected": 5.19863224029541,
      "logps/chosen": -469.70855712890625,
      "logps/rejected": -529.8267822265625,
      "loss": 0.5422,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.916710615158081,
      "rewards/margins": 0.6288083791732788,
      "rewards/rejected": -3.545518398284912,
      "step": 1340
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1277334291351145e-07,
      "logits/chosen": 5.23069953918457,
      "logits/rejected": 5.482142925262451,
      "logps/chosen": -469.21307373046875,
      "logps/rejected": -589.8045654296875,
      "loss": 0.5183,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.9651408195495605,
      "rewards/margins": 0.8228071331977844,
      "rewards/rejected": -3.7879481315612793,
      "step": 1350
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.089239692954701e-07,
      "logits/chosen": 4.438393592834473,
      "logits/rejected": 5.2009711265563965,
      "logps/chosen": -496.31463623046875,
      "logps/rejected": -561.5128173828125,
      "loss": 0.5497,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.9592137336730957,
      "rewards/margins": 0.6417844891548157,
      "rewards/rejected": -3.6009984016418457,
      "step": 1360
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.051230576558127e-07,
      "logits/chosen": 5.329151630401611,
      "logits/rejected": 5.1115946769714355,
      "logps/chosen": -510.32379150390625,
      "logps/rejected": -633.5717163085938,
      "loss": 0.5281,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.305920124053955,
      "rewards/margins": 0.6949362754821777,
      "rewards/rejected": -4.000856399536133,
      "step": 1370
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0137191367132078e-07,
      "logits/chosen": 4.635811805725098,
      "logits/rejected": 5.594052314758301,
      "logps/chosen": -538.316650390625,
      "logps/rejected": -593.9783325195312,
      "loss": 0.5179,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.257719039916992,
      "rewards/margins": 0.8566193580627441,
      "rewards/rejected": -4.1143388748168945,
      "step": 1380
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.76718259227532e-08,
      "logits/chosen": 5.040810585021973,
      "logits/rejected": 5.872241973876953,
      "logps/chosen": -559.94140625,
      "logps/rejected": -600.4876098632812,
      "loss": 0.5346,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.3652939796447754,
      "rewards/margins": 0.7677735090255737,
      "rewards/rejected": -4.1330671310424805,
      "step": 1390
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.402406545219676e-08,
      "logits/chosen": 5.209423542022705,
      "logits/rejected": 5.7783403396606445,
      "logps/chosen": -570.754150390625,
      "logps/rejected": -631.2447509765625,
      "loss": 0.5343,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.6044132709503174,
      "rewards/margins": 0.8434345126152039,
      "rewards/rejected": -4.447848320007324,
      "step": 1400
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.042988532644249e-08,
      "logits/chosen": 5.212072372436523,
      "logits/rejected": 5.8972063064575195,
      "logps/chosen": -583.494873046875,
      "logps/rejected": -651.96630859375,
      "loss": 0.5139,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.3433117866516113,
      "rewards/margins": 0.9961609840393066,
      "rewards/rejected": -4.339472770690918,
      "step": 1410
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.689052020653592e-08,
      "logits/chosen": 4.78833532333374,
      "logits/rejected": 5.89553165435791,
      "logps/chosen": -561.2659912109375,
      "logps/rejected": -631.704833984375,
      "loss": 0.5514,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.425673007965088,
      "rewards/margins": 0.8838043212890625,
      "rewards/rejected": -4.309477806091309,
      "step": 1420
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.340718592365037e-08,
      "logits/chosen": 5.993227958679199,
      "logits/rejected": 6.045858860015869,
      "logps/chosen": -575.0015869140625,
      "logps/rejected": -656.5886840820312,
      "loss": 0.5134,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -3.8804218769073486,
      "rewards/margins": 0.6545805931091309,
      "rewards/rejected": -4.535001754760742,
      "step": 1430
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.998107906142839e-08,
      "logits/chosen": 5.676799774169922,
      "logits/rejected": 6.391809940338135,
      "logps/chosen": -568.3199462890625,
      "logps/rejected": -637.5294189453125,
      "loss": 0.5254,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.7702269554138184,
      "rewards/margins": 0.8325515985488892,
      "rewards/rejected": -4.602778434753418,
      "step": 1440
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.661337654493575e-08,
      "logits/chosen": 5.199474334716797,
      "logits/rejected": 5.9724626541137695,
      "logps/chosen": -604.0126953125,
      "logps/rejected": -679.3408203125,
      "loss": 0.5511,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.842377185821533,
      "rewards/margins": 0.8208445310592651,
      "rewards/rejected": -4.663221836090088,
      "step": 1450
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.330523523636751e-08,
      "logits/chosen": 4.888028144836426,
      "logits/rejected": 5.282634258270264,
      "logps/chosen": -574.2249755859375,
      "logps/rejected": -649.12158203125,
      "loss": 0.5353,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.574962615966797,
      "rewards/margins": 0.7398016452789307,
      "rewards/rejected": -4.314764499664307,
      "step": 1460
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.005779153764682e-08,
      "logits/chosen": 5.4382123947143555,
      "logits/rejected": 5.483049392700195,
      "logps/chosen": -550.1482543945312,
      "logps/rejected": -629.5477294921875,
      "loss": 0.5274,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.7003333568573,
      "rewards/margins": 0.6894980669021606,
      "rewards/rejected": -4.389832019805908,
      "step": 1470
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.687216100005138e-08,
      "logits/chosen": 5.554679870605469,
      "logits/rejected": 6.150310516357422,
      "logps/chosen": -604.5882568359375,
      "logps/rejected": -694.1824340820312,
      "loss": 0.5241,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -3.837186336517334,
      "rewards/margins": 0.8338931202888489,
      "rewards/rejected": -4.671080589294434,
      "step": 1480
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.374943794100349e-08,
      "logits/chosen": 4.995267868041992,
      "logits/rejected": 6.089870452880859,
      "logps/chosen": -540.8165283203125,
      "logps/rejected": -621.503662109375,
      "loss": 0.5181,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.447101593017578,
      "rewards/margins": 0.998428463935852,
      "rewards/rejected": -4.445529937744141,
      "step": 1490
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.069069506815325e-08,
      "logits/chosen": 5.665683269500732,
      "logits/rejected": 5.919187545776367,
      "logps/chosen": -557.50341796875,
      "logps/rejected": -635.5902099609375,
      "loss": 0.5597,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -3.7898292541503906,
      "rewards/margins": 0.7044454216957092,
      "rewards/rejected": -4.494275093078613,
      "step": 1500
    },
    {
      "epoch": 0.8,
      "eval_logits/chosen": 6.629395008087158,
      "eval_logits/rejected": 6.464417934417725,
      "eval_logps/chosen": -596.9036865234375,
      "eval_logps/rejected": -659.0733032226562,
      "eval_loss": 0.5820685625076294,
      "eval_rewards/accuracies": 0.6959397792816162,
      "eval_rewards/chosen": -4.79062557220459,
      "eval_rewards/margins": 0.5311391949653625,
      "eval_rewards/rejected": -5.321763515472412,
      "eval_runtime": 2832.2152,
      "eval_samples_per_second": 12.373,
      "eval_steps_per_second": 0.387,
      "step": 1500
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.7696983110885746e-08,
      "logits/chosen": 6.144343376159668,
      "logits/rejected": 6.1946892738342285,
      "logps/chosen": -568.4108276367188,
      "logps/rejected": -656.3994750976562,
      "loss": 0.5713,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.8781726360321045,
      "rewards/margins": 0.7777887582778931,
      "rewards/rejected": -4.655961036682129,
      "step": 1510
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.47693304593777e-08,
      "logits/chosen": 5.610565185546875,
      "logits/rejected": 6.231629371643066,
      "logps/chosen": -578.0348510742188,
      "logps/rejected": -634.0302124023438,
      "loss": 0.5254,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.640714168548584,
      "rewards/margins": 0.8447046279907227,
      "rewards/rejected": -4.485418796539307,
      "step": 1520
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.190874281132851e-08,
      "logits/chosen": 5.386014938354492,
      "logits/rejected": 5.870030403137207,
      "logps/chosen": -544.43212890625,
      "logps/rejected": -605.0162963867188,
      "loss": 0.5313,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.533620834350586,
      "rewards/margins": 0.7047785520553589,
      "rewards/rejected": -4.238399505615234,
      "step": 1530
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.9116202826486045e-08,
      "logits/chosen": 5.818629741668701,
      "logits/rejected": 6.154656410217285,
      "logps/chosen": -583.7796630859375,
      "logps/rejected": -635.4794921875,
      "loss": 0.5337,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -3.777665615081787,
      "rewards/margins": 0.6795842051506042,
      "rewards/rejected": -4.457250118255615,
      "step": 1540
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.639266978908676e-08,
      "logits/chosen": 5.711030006408691,
      "logits/rejected": 6.358799934387207,
      "logps/chosen": -611.2662353515625,
      "logps/rejected": -676.8931884765625,
      "loss": 0.5187,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -3.886317491531372,
      "rewards/margins": 0.7436650991439819,
      "rewards/rejected": -4.629981994628906,
      "step": 1550
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.373907927832513e-08,
      "logits/chosen": 5.366969108581543,
      "logits/rejected": 5.813288688659668,
      "logps/chosen": -553.2315673828125,
      "logps/rejected": -654.0559692382812,
      "loss": 0.5274,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.5649421215057373,
      "rewards/margins": 0.8516329526901245,
      "rewards/rejected": -4.416574954986572,
      "step": 1560
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.115634284696698e-08,
      "logits/chosen": 5.473841667175293,
      "logits/rejected": 6.142959117889404,
      "logps/chosen": -595.4747314453125,
      "logps/rejected": -675.5442504882812,
      "loss": 0.4995,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.8835277557373047,
      "rewards/margins": 0.8822458386421204,
      "rewards/rejected": -4.765774250030518,
      "step": 1570
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.864534770821559e-08,
      "logits/chosen": 5.575949668884277,
      "logits/rejected": 6.3084797859191895,
      "logps/chosen": -557.6798706054688,
      "logps/rejected": -628.6642456054688,
      "loss": 0.5549,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.6573452949523926,
      "rewards/margins": 0.8320104479789734,
      "rewards/rejected": -4.489355564117432,
      "step": 1580
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.620695643093924e-08,
      "logits/chosen": 5.1766839027404785,
      "logits/rejected": 6.165072441101074,
      "logps/chosen": -578.5045166015625,
      "logps/rejected": -645.0850830078125,
      "loss": 0.5174,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.7886593341827393,
      "rewards/margins": 0.9527637362480164,
      "rewards/rejected": -4.7414231300354,
      "step": 1590
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.384200664336412e-08,
      "logits/chosen": 5.577385425567627,
      "logits/rejected": 6.328660011291504,
      "logps/chosen": -591.4779052734375,
      "logps/rejected": -644.4967041015625,
      "loss": 0.5339,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.7442920207977295,
      "rewards/margins": 0.8128402829170227,
      "rewards/rejected": -4.557132244110107,
      "step": 1600
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.155131074533529e-08,
      "logits/chosen": 4.875297546386719,
      "logits/rejected": 5.868630886077881,
      "logps/chosen": -590.7601318359375,
      "logps/rejected": -644.1740112304688,
      "loss": 0.5532,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.6774280071258545,
      "rewards/margins": 0.69183349609375,
      "rewards/rejected": -4.369261741638184,
      "step": 1610
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9335655629243645e-08,
      "logits/chosen": 5.1115217208862305,
      "logits/rejected": 5.944631099700928,
      "logps/chosen": -572.9088134765625,
      "logps/rejected": -635.9785766601562,
      "loss": 0.5635,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.67173433303833,
      "rewards/margins": 0.6962157487869263,
      "rewards/rejected": -4.367949962615967,
      "step": 1620
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.7195802409715197e-08,
      "logits/chosen": 5.198755741119385,
      "logits/rejected": 5.86753511428833,
      "logps/chosen": -608.7496948242188,
      "logps/rejected": -661.3677978515625,
      "loss": 0.5482,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.853783130645752,
      "rewards/margins": 0.7783033847808838,
      "rewards/rejected": -4.632086753845215,
      "step": 1630
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.513248616215527e-08,
      "logits/chosen": 4.92915153503418,
      "logits/rejected": 5.753048896789551,
      "logps/chosen": -574.3338623046875,
      "logps/rejected": -647.4317626953125,
      "loss": 0.5148,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.5955843925476074,
      "rewards/margins": 0.8324259519577026,
      "rewards/rejected": -4.428009986877441,
      "step": 1640
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.31464156702382e-08,
      "logits/chosen": 4.705880165100098,
      "logits/rejected": 5.527700901031494,
      "logps/chosen": -576.8226928710938,
      "logps/rejected": -628.70849609375,
      "loss": 0.5499,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.532841205596924,
      "rewards/margins": 0.7848026752471924,
      "rewards/rejected": -4.317643642425537,
      "step": 1650
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1238273182427933e-08,
      "logits/chosen": 5.681646347045898,
      "logits/rejected": 6.597903251647949,
      "logps/chosen": -569.5862426757812,
      "logps/rejected": -620.6526489257812,
      "loss": 0.5361,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.6919121742248535,
      "rewards/margins": 0.7138382196426392,
      "rewards/rejected": -4.405750274658203,
      "step": 1660
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9408714177614306e-08,
      "logits/chosen": 5.2946391105651855,
      "logits/rejected": 6.044753074645996,
      "logps/chosen": -564.3060302734375,
      "logps/rejected": -626.62353515625,
      "loss": 0.5135,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.511505126953125,
      "rewards/margins": 0.8757253885269165,
      "rewards/rejected": -4.38723087310791,
      "step": 1670
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7658367139945228e-08,
      "logits/chosen": 5.4068779945373535,
      "logits/rejected": 5.994559288024902,
      "logps/chosen": -599.1517333984375,
      "logps/rejected": -650.2825317382812,
      "loss": 0.5239,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.7978172302246094,
      "rewards/margins": 0.7726759910583496,
      "rewards/rejected": -4.570493221282959,
      "step": 1680
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5987833342931745e-08,
      "logits/chosen": 5.3490800857543945,
      "logits/rejected": 6.137597560882568,
      "logps/chosen": -583.5009155273438,
      "logps/rejected": -632.0625,
      "loss": 0.5467,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.8200180530548096,
      "rewards/margins": 0.7188345193862915,
      "rewards/rejected": -4.538852691650391,
      "step": 1690
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.439768664290053e-08,
      "logits/chosen": 5.457294940948486,
      "logits/rejected": 5.772470474243164,
      "logps/chosen": -603.9612426757812,
      "logps/rejected": -673.6254272460938,
      "loss": 0.5239,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.798058032989502,
      "rewards/margins": 0.787087619304657,
      "rewards/rejected": -4.585144996643066,
      "step": 1700
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2888473281864597e-08,
      "logits/chosen": 5.0094194412231445,
      "logits/rejected": 5.467037200927734,
      "logps/chosen": -566.6424560546875,
      "logps/rejected": -654.4783935546875,
      "loss": 0.5305,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.774543046951294,
      "rewards/margins": 0.7515050172805786,
      "rewards/rejected": -4.526047706604004,
      "step": 1710
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1460711699880082e-08,
      "logits/chosen": 4.981472015380859,
      "logits/rejected": 6.010613918304443,
      "logps/chosen": -587.7794189453125,
      "logps/rejected": -658.3820190429688,
      "loss": 0.5277,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.5816383361816406,
      "rewards/margins": 0.8929538726806641,
      "rewards/rejected": -4.474592208862305,
      "step": 1720
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0114892356953397e-08,
      "logits/chosen": 4.724175453186035,
      "logits/rejected": 5.377623558044434,
      "logps/chosen": -572.1964111328125,
      "logps/rejected": -616.9322509765625,
      "loss": 0.5707,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -3.6713428497314453,
      "rewards/margins": 0.6362916231155396,
      "rewards/rejected": -4.3076348304748535,
      "step": 1730
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.851477564560061e-09,
      "logits/chosen": 5.281374454498291,
      "logits/rejected": 5.823667526245117,
      "logps/chosen": -574.9295654296875,
      "logps/rejected": -682.01611328125,
      "loss": 0.5272,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.782670497894287,
      "rewards/margins": 1.0396969318389893,
      "rewards/rejected": -4.8223676681518555,
      "step": 1740
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.670901326832763e-09,
      "logits/chosen": 5.813240051269531,
      "logits/rejected": 6.113523006439209,
      "logps/chosen": -583.0259399414062,
      "logps/rejected": -707.0408935546875,
      "loss": 0.5068,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.896685838699341,
      "rewards/margins": 0.9632736444473267,
      "rewards/rejected": -4.859958648681641,
      "step": 1750
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.5735691914738936e-09,
      "logits/chosen": 4.764095783233643,
      "logits/rejected": 5.158614158630371,
      "logps/chosen": -563.710693359375,
      "logps/rejected": -646.1812744140625,
      "loss": 0.5635,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.6481738090515137,
      "rewards/margins": 0.7430108785629272,
      "rewards/rejected": -4.3911848068237305,
      "step": 1760
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.559858110443016e-09,
      "logits/chosen": 4.941697120666504,
      "logits/rejected": 5.857718467712402,
      "logps/chosen": -589.8416137695312,
      "logps/rejected": -651.1143798828125,
      "loss": 0.5218,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.781978130340576,
      "rewards/margins": 0.8180473446846008,
      "rewards/rejected": -4.600025177001953,
      "step": 1770
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.6301163104676685e-09,
      "logits/chosen": 5.284440517425537,
      "logits/rejected": 5.553745269775391,
      "logps/chosen": -551.4322509765625,
      "logps/rejected": -651.212646484375,
      "loss": 0.5407,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.6018593311309814,
      "rewards/margins": 0.7678863406181335,
      "rewards/rejected": -4.36974573135376,
      "step": 1780
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.784663173421438e-09,
      "logits/chosen": 5.374969959259033,
      "logits/rejected": 5.7753376960754395,
      "logps/chosen": -622.2989501953125,
      "logps/rejected": -689.6795654296875,
      "loss": 0.5308,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.73246693611145,
      "rewards/margins": 0.8092583417892456,
      "rewards/rejected": -4.5417256355285645,
      "step": 1790
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.023789126611137e-09,
      "logits/chosen": 5.826882839202881,
      "logits/rejected": 6.438210964202881,
      "logps/chosen": -580.4129638671875,
      "logps/rejected": -642.33740234375,
      "loss": 0.5188,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.765054702758789,
      "rewards/margins": 0.8300830721855164,
      "rewards/rejected": -4.595137596130371,
      "step": 1800
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3477555430100604e-09,
      "logits/chosen": 5.330014228820801,
      "logits/rejected": 6.266722679138184,
      "logps/chosen": -545.0280151367188,
      "logps/rejected": -623.4840087890625,
      "loss": 0.5255,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.382579803466797,
      "rewards/margins": 1.0013315677642822,
      "rewards/rejected": -4.383911609649658,
      "step": 1810
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.7567946514721322e-09,
      "logits/chosen": 5.459120273590088,
      "logits/rejected": 5.671516418457031,
      "logps/chosen": -565.1712036132812,
      "logps/rejected": -651.7200927734375,
      "loss": 0.5631,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.7416863441467285,
      "rewards/margins": 0.7198008894920349,
      "rewards/rejected": -4.46148681640625,
      "step": 1820
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.2511094569571668e-09,
      "logits/chosen": 4.77047061920166,
      "logits/rejected": 5.680732727050781,
      "logps/chosen": -540.1213989257812,
      "logps/rejected": -612.4068603515625,
      "loss": 0.5294,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.508056640625,
      "rewards/margins": 0.8371505737304688,
      "rewards/rejected": -4.345207214355469,
      "step": 1830
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.308736707954289e-10,
      "logits/chosen": 5.112320899963379,
      "logits/rejected": 6.371096611022949,
      "logps/chosen": -591.5335693359375,
      "logps/rejected": -622.4219360351562,
      "loss": 0.5351,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.7684085369110107,
      "rewards/margins": 0.8042453527450562,
      "rewards/rejected": -4.572653770446777,
      "step": 1840
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.962316510149222e-10,
      "logits/chosen": 4.924439430236816,
      "logits/rejected": 5.723077774047852,
      "logps/chosen": -528.1791381835938,
      "logps/rejected": -613.794189453125,
      "loss": 0.5155,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.5361733436584473,
      "rewards/margins": 0.8941107988357544,
      "rewards/rejected": -4.43028450012207,
      "step": 1850
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.4729835275189016e-10,
      "logits/chosen": 5.2165093421936035,
      "logits/rejected": 5.809523105621338,
      "logps/chosen": -512.5826416015625,
      "logps/rejected": -562.5357055664062,
      "loss": 0.5279,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.3880043029785156,
      "rewards/margins": 0.6139224767684937,
      "rewards/rejected": -4.001926898956299,
      "step": 1860
    },
    {
      "epoch": 0.99,
      "learning_rate": 8.415928876176482e-11,
      "logits/chosen": 5.226788520812988,
      "logits/rejected": 5.463950157165527,
      "logps/chosen": -568.6761474609375,
      "logps/rejected": -642.3060302734375,
      "loss": 0.5796,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -3.871469020843506,
      "rewards/margins": 0.6007556319236755,
      "rewards/rejected": -4.472224235534668,
      "step": 1870
    },
    {
      "epoch": 1.0,
      "learning_rate": 6.870500044303673e-12,
      "logits/chosen": 5.113773345947266,
      "logits/rejected": 5.360236167907715,
      "logps/chosen": -532.0494995117188,
      "logps/rejected": -629.88818359375,
      "loss": 0.537,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.353315830230713,
      "rewards/margins": 0.727187991142273,
      "rewards/rejected": -4.080503940582275,
      "step": 1880
    },
    {
      "epoch": 1.0,
      "step": 1884,
      "total_flos": 0.0,
      "train_loss": 0.5658232939218006,
      "train_runtime": 45309.3847,
      "train_samples_per_second": 2.662,
      "train_steps_per_second": 0.042
    }
  ],
  "logging_steps": 10,
  "max_steps": 1884,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}