{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.999297541394882,
  "eval_steps": 400,
  "global_step": 5604,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002676032781401572,
      "grad_norm": 4.174769404980618,
      "learning_rate": 8.9126559714795e-09,
      "logits/chosen": -0.056673564016819,
      "logits/rejected": 0.15200209617614746,
      "logps/chosen": -1.7160017490386963,
      "logps/rejected": -1.8894321918487549,
      "loss": 0.2436,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.7160017490386963,
      "rewards/margins": 0.17343027889728546,
      "rewards/rejected": -1.8894321918487549,
      "step": 5
    },
    {
      "epoch": 0.005352065562803144,
      "grad_norm": 2.8626364007215304,
      "learning_rate": 1.7825311942959e-08,
      "logits/chosen": 0.015492829494178295,
      "logits/rejected": 0.1357966512441635,
      "logps/chosen": -1.8026939630508423,
      "logps/rejected": -1.846070647239685,
      "loss": 0.25,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.8026939630508423,
      "rewards/margins": 0.043376728892326355,
      "rewards/rejected": -1.846070647239685,
      "step": 10
    },
    {
      "epoch": 0.008028098344204716,
      "grad_norm": 3.044428420168379,
      "learning_rate": 2.67379679144385e-08,
      "logits/chosen": -0.03407303988933563,
      "logits/rejected": 0.06202689930796623,
      "logps/chosen": -1.6353429555892944,
      "logps/rejected": -1.765667200088501,
      "loss": 0.287,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.6353429555892944,
      "rewards/margins": 0.13032427430152893,
      "rewards/rejected": -1.765667200088501,
      "step": 15
    },
    {
      "epoch": 0.010704131125606288,
      "grad_norm": 3.8909506975846084,
      "learning_rate": 3.5650623885918e-08,
      "logits/chosen": -0.049756430089473724,
      "logits/rejected": 0.0334530733525753,
      "logps/chosen": -1.7236230373382568,
      "logps/rejected": -1.8058974742889404,
      "loss": 0.2784,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.7236230373382568,
      "rewards/margins": 0.08227436244487762,
      "rewards/rejected": -1.8058974742889404,
      "step": 20
    },
    {
      "epoch": 0.013380163907007862,
      "grad_norm": 3.7378465518883943,
      "learning_rate": 4.45632798573975e-08,
      "logits/chosen": -0.05973156541585922,
      "logits/rejected": 0.024282705038785934,
      "logps/chosen": -1.8695319890975952,
      "logps/rejected": -1.7784847021102905,
      "loss": 0.3067,
      "rewards/accuracies": 0.3812499940395355,
      "rewards/chosen": -1.8695319890975952,
      "rewards/margins": -0.09104712307453156,
      "rewards/rejected": -1.7784847021102905,
      "step": 25
    },
    {
      "epoch": 0.016056196688409432,
      "grad_norm": 2.9924518409881413,
      "learning_rate": 5.3475935828877e-08,
      "logits/chosen": -0.07078413665294647,
      "logits/rejected": 0.02136463299393654,
      "logps/chosen": -1.907697319984436,
      "logps/rejected": -1.8313062191009521,
      "loss": 0.2477,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -1.907697319984436,
      "rewards/margins": -0.07639118283987045,
      "rewards/rejected": -1.8313062191009521,
      "step": 30
    },
    {
      "epoch": 0.018732229469811006,
      "grad_norm": 3.9037635937417243,
      "learning_rate": 6.23885918003565e-08,
      "logits/chosen": -0.05222669243812561,
      "logits/rejected": 0.10753919184207916,
      "logps/chosen": -1.8471834659576416,
      "logps/rejected": -1.9986279010772705,
      "loss": 0.2626,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.8471834659576416,
      "rewards/margins": 0.15144455432891846,
      "rewards/rejected": -1.9986279010772705,
      "step": 35
    },
    {
      "epoch": 0.021408262251212576,
      "grad_norm": 3.260061146078127,
      "learning_rate": 7.1301247771836e-08,
      "logits/chosen": 0.04427720606327057,
      "logits/rejected": 0.21839460730552673,
      "logps/chosen": -1.886055588722229,
      "logps/rejected": -1.7470966577529907,
      "loss": 0.2781,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -1.886055588722229,
      "rewards/margins": -0.1389588713645935,
      "rewards/rejected": -1.7470966577529907,
      "step": 40
    },
    {
      "epoch": 0.02408429503261415,
      "grad_norm": 3.7651938269189533,
      "learning_rate": 8.021390374331551e-08,
      "logits/chosen": 0.019146930426359177,
      "logits/rejected": 0.21615441143512726,
      "logps/chosen": -1.8423458337783813,
      "logps/rejected": -1.8769302368164062,
      "loss": 0.2696,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.8423458337783813,
      "rewards/margins": 0.03458467125892639,
      "rewards/rejected": -1.8769302368164062,
      "step": 45
    },
    {
      "epoch": 0.026760327814015723,
      "grad_norm": 3.566135479934334,
      "learning_rate": 8.9126559714795e-08,
      "logits/chosen": -0.050375960767269135,
      "logits/rejected": 0.09876345098018646,
      "logps/chosen": -1.9080318212509155,
      "logps/rejected": -1.7855631113052368,
      "loss": 0.2647,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.9080318212509155,
      "rewards/margins": -0.12246876955032349,
      "rewards/rejected": -1.7855631113052368,
      "step": 50
    },
    {
      "epoch": 0.029436360595417294,
      "grad_norm": 3.2091482239918308,
      "learning_rate": 9.80392156862745e-08,
      "logits/chosen": -0.10335006564855576,
      "logits/rejected": 0.11803705990314484,
      "logps/chosen": -1.8465286493301392,
      "logps/rejected": -1.8804638385772705,
      "loss": 0.2517,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.8465286493301392,
      "rewards/margins": 0.03393523767590523,
      "rewards/rejected": -1.8804638385772705,
      "step": 55
    },
    {
      "epoch": 0.032112393376818864,
      "grad_norm": 3.3346635910473887,
      "learning_rate": 1.06951871657754e-07,
      "logits/chosen": -0.09414264559745789,
      "logits/rejected": 0.09447915107011795,
      "logps/chosen": -1.8043931722640991,
      "logps/rejected": -1.9090369939804077,
      "loss": 0.2443,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.8043931722640991,
      "rewards/margins": 0.10464388132095337,
      "rewards/rejected": -1.9090369939804077,
      "step": 60
    },
    {
      "epoch": 0.03478842615822044,
      "grad_norm": 3.1364454101958548,
      "learning_rate": 1.158645276292335e-07,
      "logits/chosen": -0.02966163121163845,
      "logits/rejected": 0.11632075160741806,
      "logps/chosen": -1.6551532745361328,
      "logps/rejected": -1.788581132888794,
      "loss": 0.2675,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.6551532745361328,
      "rewards/margins": 0.13342775404453278,
      "rewards/rejected": -1.788581132888794,
      "step": 65
    },
    {
      "epoch": 0.03746445893962201,
      "grad_norm": 4.538924165904268,
      "learning_rate": 1.24777183600713e-07,
      "logits/chosen": -0.06675219535827637,
      "logits/rejected": 0.085882268846035,
      "logps/chosen": -1.7921323776245117,
      "logps/rejected": -1.8404977321624756,
      "loss": 0.2704,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -1.7921323776245117,
      "rewards/margins": 0.04836542159318924,
      "rewards/rejected": -1.8404977321624756,
      "step": 70
    },
    {
      "epoch": 0.04014049172102358,
      "grad_norm": 2.8899566254010987,
      "learning_rate": 1.3368983957219251e-07,
      "logits/chosen": -0.04987819120287895,
      "logits/rejected": 0.1338050663471222,
      "logps/chosen": -1.822100043296814,
      "logps/rejected": -2.090280771255493,
      "loss": 0.238,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.822100043296814,
      "rewards/margins": 0.26818060874938965,
      "rewards/rejected": -2.090280771255493,
      "step": 75
    },
    {
      "epoch": 0.04281652450242515,
      "grad_norm": 2.7037524466902587,
      "learning_rate": 1.42602495543672e-07,
      "logits/chosen": 0.008540439419448376,
      "logits/rejected": 0.11213047802448273,
      "logps/chosen": -1.7725322246551514,
      "logps/rejected": -1.8043262958526611,
      "loss": 0.2662,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.7725322246551514,
      "rewards/margins": 0.03179407864809036,
      "rewards/rejected": -1.8043262958526611,
      "step": 80
    },
    {
      "epoch": 0.04549255728382673,
      "grad_norm": 2.753588303143848,
      "learning_rate": 1.5151515151515152e-07,
      "logits/chosen": -0.14115747809410095,
      "logits/rejected": 0.11033763736486435,
      "logps/chosen": -1.8558228015899658,
      "logps/rejected": -2.0468955039978027,
      "loss": 0.2628,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.8558228015899658,
      "rewards/margins": 0.19107282161712646,
      "rewards/rejected": -2.0468955039978027,
      "step": 85
    },
    {
      "epoch": 0.0481685900652283,
      "grad_norm": 2.4810036120937773,
      "learning_rate": 1.6042780748663102e-07,
      "logits/chosen": 0.09177587926387787,
      "logits/rejected": 0.05044828727841377,
      "logps/chosen": -1.8309383392333984,
      "logps/rejected": -1.8402868509292603,
      "loss": 0.2741,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -1.8309383392333984,
      "rewards/margins": 0.009348212741315365,
      "rewards/rejected": -1.8402868509292603,
      "step": 90
    },
    {
      "epoch": 0.05084462284662987,
      "grad_norm": 2.5214005774541346,
      "learning_rate": 1.693404634581105e-07,
      "logits/chosen": -0.05386476591229439,
      "logits/rejected": 0.09711028635501862,
      "logps/chosen": -1.915281891822815,
      "logps/rejected": -1.996042251586914,
      "loss": 0.2471,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.915281891822815,
      "rewards/margins": 0.08076085150241852,
      "rewards/rejected": -1.996042251586914,
      "step": 95
    },
    {
      "epoch": 0.05352065562803145,
      "grad_norm": 2.386081290272195,
      "learning_rate": 1.7825311942959e-07,
      "logits/chosen": -0.04361366480588913,
      "logits/rejected": 0.017345154657959938,
      "logps/chosen": -1.7915188074111938,
      "logps/rejected": -1.9044063091278076,
      "loss": 0.2475,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.7915188074111938,
      "rewards/margins": 0.11288763582706451,
      "rewards/rejected": -1.9044063091278076,
      "step": 100
    },
    {
      "epoch": 0.05619668840943302,
      "grad_norm": 2.3355814398098738,
      "learning_rate": 1.8716577540106952e-07,
      "logits/chosen": 0.05401003360748291,
      "logits/rejected": 0.08016423881053925,
      "logps/chosen": -1.775671362876892,
      "logps/rejected": -1.9383106231689453,
      "loss": 0.2496,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.775671362876892,
      "rewards/margins": 0.16263923048973083,
      "rewards/rejected": -1.9383106231689453,
      "step": 105
    },
    {
      "epoch": 0.05887272119083459,
      "grad_norm": 2.5055549833177158,
      "learning_rate": 1.96078431372549e-07,
      "logits/chosen": 0.026605969294905663,
      "logits/rejected": 0.12203983962535858,
      "logps/chosen": -1.8630073070526123,
      "logps/rejected": -1.9225571155548096,
      "loss": 0.2607,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.8630073070526123,
      "rewards/margins": 0.05954958125948906,
      "rewards/rejected": -1.9225571155548096,
      "step": 110
    },
    {
      "epoch": 0.06154875397223616,
      "grad_norm": 2.42093904789979,
      "learning_rate": 2.049910873440285e-07,
      "logits/chosen": 0.04268549010157585,
      "logits/rejected": 0.2566087543964386,
      "logps/chosen": -1.8545005321502686,
      "logps/rejected": -2.1896767616271973,
      "loss": 0.2134,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.8545005321502686,
      "rewards/margins": 0.3351760804653168,
      "rewards/rejected": -2.1896767616271973,
      "step": 115
    },
    {
      "epoch": 0.06422478675363773,
      "grad_norm": 1.8206023092375123,
      "learning_rate": 2.13903743315508e-07,
      "logits/chosen": -0.06773766130208969,
      "logits/rejected": 0.1087050586938858,
      "logps/chosen": -2.0017330646514893,
      "logps/rejected": -2.148353099822998,
      "loss": 0.2308,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -2.0017330646514893,
      "rewards/margins": 0.14662012457847595,
      "rewards/rejected": -2.148353099822998,
      "step": 120
    },
    {
      "epoch": 0.0669008195350393,
      "grad_norm": 2.8040060365210007,
      "learning_rate": 2.2281639928698751e-07,
      "logits/chosen": -0.0807386040687561,
      "logits/rejected": 0.05026230961084366,
      "logps/chosen": -1.89572274684906,
      "logps/rejected": -1.8105024099349976,
      "loss": 0.2647,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.89572274684906,
      "rewards/margins": -0.08522024005651474,
      "rewards/rejected": -1.8105024099349976,
      "step": 125
    },
    {
      "epoch": 0.06957685231644088,
      "grad_norm": 2.6288404494949145,
      "learning_rate": 2.31729055258467e-07,
      "logits/chosen": 0.03656303137540817,
      "logits/rejected": 0.1726987063884735,
      "logps/chosen": -1.9655530452728271,
      "logps/rejected": -2.089103937149048,
      "loss": 0.231,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -1.9655530452728271,
      "rewards/margins": 0.12355075031518936,
      "rewards/rejected": -2.089103937149048,
      "step": 130
    },
    {
      "epoch": 0.07225288509784245,
      "grad_norm": 2.087737893476425,
      "learning_rate": 2.406417112299465e-07,
      "logits/chosen": -0.019556980580091476,
      "logits/rejected": 0.10337207466363907,
      "logps/chosen": -2.0496957302093506,
      "logps/rejected": -2.0199496746063232,
      "loss": 0.241,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.0496957302093506,
      "rewards/margins": -0.02974613569676876,
      "rewards/rejected": -2.0199496746063232,
      "step": 135
    },
    {
      "epoch": 0.07492891787924402,
      "grad_norm": 2.9049109285511663,
      "learning_rate": 2.49554367201426e-07,
      "logits/chosen": -0.025853702798485756,
      "logits/rejected": 0.14452826976776123,
      "logps/chosen": -2.0190365314483643,
      "logps/rejected": -2.2830276489257812,
      "loss": 0.2095,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -2.0190365314483643,
      "rewards/margins": 0.2639915347099304,
      "rewards/rejected": -2.2830276489257812,
      "step": 140
    },
    {
      "epoch": 0.0776049506606456,
      "grad_norm": 2.156959064207746,
      "learning_rate": 2.5846702317290554e-07,
      "logits/chosen": -0.011653195135295391,
      "logits/rejected": 0.14680761098861694,
      "logps/chosen": -1.9980818033218384,
      "logps/rejected": -2.149132251739502,
      "loss": 0.2268,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -1.9980818033218384,
      "rewards/margins": 0.1510503739118576,
      "rewards/rejected": -2.149132251739502,
      "step": 145
    },
    {
      "epoch": 0.08028098344204716,
      "grad_norm": 2.3619427872695433,
      "learning_rate": 2.6737967914438503e-07,
      "logits/chosen": -0.034545183181762695,
      "logits/rejected": 0.13397535681724548,
      "logps/chosen": -1.9922664165496826,
      "logps/rejected": -1.99588143825531,
      "loss": 0.2493,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.9922664165496826,
      "rewards/margins": 0.003614726709201932,
      "rewards/rejected": -1.99588143825531,
      "step": 150
    },
    {
      "epoch": 0.08295701622344874,
      "grad_norm": 2.2327996873434297,
      "learning_rate": 2.762923351158645e-07,
      "logits/chosen": -0.0200195349752903,
      "logits/rejected": 0.026785830035805702,
      "logps/chosen": -2.1182286739349365,
      "logps/rejected": -2.162487030029297,
      "loss": 0.2351,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.1182286739349365,
      "rewards/margins": 0.0442582368850708,
      "rewards/rejected": -2.162487030029297,
      "step": 155
    },
    {
      "epoch": 0.0856330490048503,
      "grad_norm": 2.1191289655384895,
      "learning_rate": 2.85204991087344e-07,
      "logits/chosen": -0.12646076083183289,
      "logits/rejected": 0.01917138509452343,
      "logps/chosen": -2.3306081295013428,
      "logps/rejected": -2.306840181350708,
      "loss": 0.2229,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.3306081295013428,
      "rewards/margins": -0.023767992854118347,
      "rewards/rejected": -2.306840181350708,
      "step": 160
    },
    {
      "epoch": 0.08830908178625188,
      "grad_norm": 2.04617266810806,
      "learning_rate": 2.941176470588235e-07,
      "logits/chosen": -0.04032761603593826,
      "logits/rejected": 0.14231903851032257,
      "logps/chosen": -2.0537381172180176,
      "logps/rejected": -2.376739978790283,
      "loss": 0.2268,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -2.0537381172180176,
      "rewards/margins": 0.32300177216529846,
      "rewards/rejected": -2.376739978790283,
      "step": 165
    },
    {
      "epoch": 0.09098511456765346,
      "grad_norm": 2.009380146648297,
      "learning_rate": 3.0303030303030305e-07,
      "logits/chosen": -0.05518675968050957,
      "logits/rejected": 0.0020436719059944153,
      "logps/chosen": -2.344125509262085,
      "logps/rejected": -2.3139758110046387,
      "loss": 0.2106,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -2.344125509262085,
      "rewards/margins": -0.030149567872285843,
      "rewards/rejected": -2.3139758110046387,
      "step": 170
    },
    {
      "epoch": 0.09366114734905502,
      "grad_norm": 2.018083114120056,
      "learning_rate": 3.1194295900178254e-07,
      "logits/chosen": 0.09154976904392242,
      "logits/rejected": 0.09093359112739563,
      "logps/chosen": -2.2235960960388184,
      "logps/rejected": -2.286865711212158,
      "loss": 0.2469,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -2.2235960960388184,
      "rewards/margins": 0.06326953321695328,
      "rewards/rejected": -2.286865711212158,
      "step": 175
    },
    {
      "epoch": 0.0963371801304566,
      "grad_norm": 1.701656302291715,
      "learning_rate": 3.2085561497326203e-07,
      "logits/chosen": 0.03326406702399254,
      "logits/rejected": 0.03350166231393814,
      "logps/chosen": -2.326598644256592,
      "logps/rejected": -2.278094530105591,
      "loss": 0.2179,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.326598644256592,
      "rewards/margins": -0.048504143953323364,
      "rewards/rejected": -2.278094530105591,
      "step": 180
    },
    {
      "epoch": 0.09901321291185818,
      "grad_norm": 2.064670625121251,
      "learning_rate": 3.297682709447415e-07,
      "logits/chosen": -0.10991539061069489,
      "logits/rejected": -0.01808272674679756,
      "logps/chosen": -2.2595760822296143,
      "logps/rejected": -2.352756977081299,
      "loss": 0.2346,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -2.2595760822296143,
      "rewards/margins": 0.09318114817142487,
      "rewards/rejected": -2.352756977081299,
      "step": 185
    },
    {
      "epoch": 0.10168924569325974,
      "grad_norm": 2.8550580867543345,
      "learning_rate": 3.38680926916221e-07,
      "logits/chosen": -0.016482654958963394,
      "logits/rejected": 0.12248317152261734,
      "logps/chosen": -2.8278183937072754,
      "logps/rejected": -2.660628318786621,
      "loss": 0.194,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -2.8278183937072754,
      "rewards/margins": -0.16718989610671997,
      "rewards/rejected": -2.660628318786621,
      "step": 190
    },
    {
      "epoch": 0.10436527847466132,
      "grad_norm": 2.4580704082500793,
      "learning_rate": 3.475935828877005e-07,
      "logits/chosen": 0.03471102565526962,
      "logits/rejected": 0.20238180458545685,
      "logps/chosen": -2.1727375984191895,
      "logps/rejected": -2.2292609214782715,
      "loss": 0.2117,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.1727375984191895,
      "rewards/margins": 0.05652397871017456,
      "rewards/rejected": -2.2292609214782715,
      "step": 195
    },
    {
      "epoch": 0.1070413112560629,
      "grad_norm": 3.151318863169166,
      "learning_rate": 3.5650623885918e-07,
      "logits/chosen": -0.037284087389707565,
      "logits/rejected": 0.11781023442745209,
      "logps/chosen": -2.721318244934082,
      "logps/rejected": -2.402465343475342,
      "loss": 0.2105,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -2.721318244934082,
      "rewards/margins": -0.31885290145874023,
      "rewards/rejected": -2.402465343475342,
      "step": 200
    },
    {
      "epoch": 0.10971734403746446,
      "grad_norm": 2.6550912450394533,
      "learning_rate": 3.654188948306595e-07,
      "logits/chosen": -0.013735070824623108,
      "logits/rejected": 0.15475720167160034,
      "logps/chosen": -3.2313952445983887,
      "logps/rejected": -2.8162760734558105,
      "loss": 0.1895,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.2313952445983887,
      "rewards/margins": -0.415119469165802,
      "rewards/rejected": -2.8162760734558105,
      "step": 205
    },
    {
      "epoch": 0.11239337681886603,
      "grad_norm": 2.2808856180721637,
      "learning_rate": 3.7433155080213904e-07,
      "logits/chosen": -0.09457657486200333,
      "logits/rejected": 0.12909016013145447,
      "logps/chosen": -3.0543413162231445,
      "logps/rejected": -3.4971511363983154,
      "loss": 0.1495,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -3.0543413162231445,
      "rewards/margins": 0.4428097605705261,
      "rewards/rejected": -3.4971511363983154,
      "step": 210
    },
    {
      "epoch": 0.1150694096002676,
      "grad_norm": 2.1192106274005464,
      "learning_rate": 3.8324420677361853e-07,
      "logits/chosen": -0.13070939481258392,
      "logits/rejected": 0.14487504959106445,
      "logps/chosen": -2.8309102058410645,
      "logps/rejected": -2.9786882400512695,
      "loss": 0.1608,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -2.8309102058410645,
      "rewards/margins": 0.14777803421020508,
      "rewards/rejected": -2.9786882400512695,
      "step": 215
    },
    {
      "epoch": 0.11774544238166917,
      "grad_norm": 2.117697527140775,
      "learning_rate": 3.92156862745098e-07,
      "logits/chosen": 0.06672084331512451,
      "logits/rejected": 0.1787821650505066,
      "logps/chosen": -3.4120097160339355,
      "logps/rejected": -3.9355578422546387,
      "loss": 0.14,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -3.4120097160339355,
      "rewards/margins": 0.5235484838485718,
      "rewards/rejected": -3.9355578422546387,
      "step": 220
    },
    {
      "epoch": 0.12042147516307075,
      "grad_norm": 1.9630938588222873,
      "learning_rate": 4.010695187165775e-07,
      "logits/chosen": -0.09861567616462708,
      "logits/rejected": 0.0825483575463295,
      "logps/chosen": -3.199176788330078,
      "logps/rejected": -3.2739837169647217,
      "loss": 0.141,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -3.199176788330078,
      "rewards/margins": 0.07480701804161072,
      "rewards/rejected": -3.2739837169647217,
      "step": 225
    },
    {
      "epoch": 0.12309750794447231,
      "grad_norm": 2.0932688158739277,
      "learning_rate": 4.09982174688057e-07,
      "logits/chosen": 0.01902366243302822,
      "logits/rejected": 0.10696601867675781,
      "logps/chosen": -3.955059766769409,
      "logps/rejected": -3.9199607372283936,
      "loss": 0.1372,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -3.955059766769409,
      "rewards/margins": -0.03509901836514473,
      "rewards/rejected": -3.9199607372283936,
      "step": 230
    },
    {
      "epoch": 0.1257735407258739,
      "grad_norm": 2.249728441941806,
      "learning_rate": 4.188948306595365e-07,
      "logits/chosen": 0.04213883727788925,
      "logits/rejected": 0.20603123307228088,
      "logps/chosen": -3.6573188304901123,
      "logps/rejected": -4.0042901039123535,
      "loss": 0.1164,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -3.6573188304901123,
      "rewards/margins": 0.346971333026886,
      "rewards/rejected": -4.0042901039123535,
      "step": 235
    },
    {
      "epoch": 0.12844957350727546,
      "grad_norm": 2.273064270761011,
      "learning_rate": 4.27807486631016e-07,
      "logits/chosen": 0.013959121890366077,
      "logits/rejected": 0.15524466335773468,
      "logps/chosen": -4.024859428405762,
      "logps/rejected": -4.038453578948975,
      "loss": 0.1286,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -4.024859428405762,
      "rewards/margins": 0.013593924231827259,
      "rewards/rejected": -4.038453578948975,
      "step": 240
    },
    {
      "epoch": 0.13112560628867703,
      "grad_norm": 1.8055229717224743,
      "learning_rate": 4.3672014260249554e-07,
      "logits/chosen": 0.10808676481246948,
      "logits/rejected": 0.25572407245635986,
      "logps/chosen": -3.702683925628662,
      "logps/rejected": -4.566058158874512,
      "loss": 0.1141,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -3.702683925628662,
      "rewards/margins": 0.8633737564086914,
      "rewards/rejected": -4.566058158874512,
      "step": 245
    },
    {
      "epoch": 0.1338016390700786,
      "grad_norm": 1.6074368965988135,
      "learning_rate": 4.4563279857397503e-07,
      "logits/chosen": 0.03807227686047554,
      "logits/rejected": 0.23177461326122284,
      "logps/chosen": -5.78887414932251,
      "logps/rejected": -5.649636268615723,
      "loss": 0.1132,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -5.78887414932251,
      "rewards/margins": -0.13923706114292145,
      "rewards/rejected": -5.649636268615723,
      "step": 250
    },
    {
      "epoch": 0.1364776718514802,
      "grad_norm": 1.251947569890329,
      "learning_rate": 4.545454545454545e-07,
      "logits/chosen": 0.06387828290462494,
      "logits/rejected": 0.24852073192596436,
      "logps/chosen": -4.003514289855957,
      "logps/rejected": -4.843715667724609,
      "loss": 0.111,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -4.003514289855957,
      "rewards/margins": 0.8402018547058105,
      "rewards/rejected": -4.843715667724609,
      "step": 255
    },
    {
      "epoch": 0.13915370463288176,
      "grad_norm": 1.0557755605880161,
      "learning_rate": 4.63458110516934e-07,
      "logits/chosen": -0.09168213605880737,
      "logits/rejected": 0.03592577204108238,
      "logps/chosen": -5.3444318771362305,
      "logps/rejected": -4.5207133293151855,
      "loss": 0.0874,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -5.3444318771362305,
      "rewards/margins": -0.8237188458442688,
      "rewards/rejected": -4.5207133293151855,
      "step": 260
    },
    {
      "epoch": 0.1418297374142833,
      "grad_norm": 0.948002934749555,
      "learning_rate": 4.723707664884135e-07,
      "logits/chosen": 0.09713621437549591,
      "logits/rejected": 0.1900687962770462,
      "logps/chosen": -6.329071998596191,
      "logps/rejected": -4.980845928192139,
      "loss": 0.0961,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -6.329071998596191,
      "rewards/margins": -1.3482263088226318,
      "rewards/rejected": -4.980845928192139,
      "step": 265
    },
    {
      "epoch": 0.1445057701956849,
      "grad_norm": 0.9982119135432448,
      "learning_rate": 4.81283422459893e-07,
      "logits/chosen": 0.031049340963363647,
      "logits/rejected": 0.20883283019065857,
      "logps/chosen": -4.965585708618164,
      "logps/rejected": -5.328085422515869,
      "loss": 0.0955,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -4.965585708618164,
      "rewards/margins": 0.3624994158744812,
      "rewards/rejected": -5.328085422515869,
      "step": 270
    },
    {
      "epoch": 0.14718180297708647,
      "grad_norm": 1.6297908390461078,
      "learning_rate": 4.901960784313725e-07,
      "logits/chosen": 0.14937344193458557,
      "logits/rejected": 0.25307050347328186,
      "logps/chosen": -5.577644348144531,
      "logps/rejected": -6.263503074645996,
      "loss": 0.1076,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -5.577644348144531,
      "rewards/margins": 0.685858428478241,
      "rewards/rejected": -6.263503074645996,
      "step": 275
    },
    {
      "epoch": 0.14985783575848804,
      "grad_norm": 1.5496631366124152,
      "learning_rate": 4.99108734402852e-07,
      "logits/chosen": 0.014246372506022453,
      "logits/rejected": 0.21954336762428284,
      "logps/chosen": -6.520092010498047,
      "logps/rejected": -6.146762847900391,
      "loss": 0.0769,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -6.520092010498047,
      "rewards/margins": -0.3733291029930115,
      "rewards/rejected": -6.146762847900391,
      "step": 280
    },
    {
      "epoch": 0.15253386853988962,
      "grad_norm": 1.64411911013092,
      "learning_rate": 5.080213903743315e-07,
      "logits/chosen": 0.06115398555994034,
      "logits/rejected": 0.2414974868297577,
      "logps/chosen": -6.048588752746582,
      "logps/rejected": -5.88077974319458,
      "loss": 0.0897,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -6.048588752746582,
      "rewards/margins": -0.16780927777290344,
      "rewards/rejected": -5.88077974319458,
      "step": 285
    },
    {
      "epoch": 0.1552099013212912,
      "grad_norm": 1.4460128553048879,
      "learning_rate": 5.169340463458111e-07,
      "logits/chosen": 0.08946628123521805,
      "logits/rejected": 0.4639100432395935,
      "logps/chosen": -5.004059791564941,
      "logps/rejected": -6.311199188232422,
      "loss": 0.0526,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -5.004059791564941,
      "rewards/margins": 1.3071396350860596,
      "rewards/rejected": -6.311199188232422,
      "step": 290
    },
    {
      "epoch": 0.15788593410269275,
      "grad_norm": 4.091528892126652,
      "learning_rate": 5.258467023172905e-07,
      "logits/chosen": 0.1946166455745697,
      "logits/rejected": 0.2628430426120758,
      "logps/chosen": -7.0160675048828125,
      "logps/rejected": -6.312521934509277,
      "loss": 0.0651,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -7.0160675048828125,
      "rewards/margins": -0.7035457491874695,
      "rewards/rejected": -6.312521934509277,
      "step": 295
    },
    {
      "epoch": 0.16056196688409433,
      "grad_norm": 4.919579238293798,
      "learning_rate": 5.347593582887701e-07,
      "logits/chosen": 0.3639879822731018,
      "logits/rejected": 0.5850847363471985,
      "logps/chosen": -8.438272476196289,
      "logps/rejected": -8.288660049438477,
      "loss": 0.041,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -8.438272476196289,
      "rewards/margins": -0.14961174130439758,
      "rewards/rejected": -8.288660049438477,
      "step": 300
    },
    {
      "epoch": 0.1632379996654959,
      "grad_norm": 1.7385314438585548,
      "learning_rate": 5.436720142602496e-07,
      "logits/chosen": 0.8661595582962036,
      "logits/rejected": 0.966326117515564,
      "logps/chosen": -10.178471565246582,
      "logps/rejected": -9.510921478271484,
      "loss": 0.0148,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -10.178471565246582,
      "rewards/margins": -0.6675509214401245,
      "rewards/rejected": -9.510921478271484,
      "step": 305
    },
    {
      "epoch": 0.16591403244689748,
      "grad_norm": 2.2433192138162146,
      "learning_rate": 5.52584670231729e-07,
      "logits/chosen": 1.4416261911392212,
      "logits/rejected": 1.5715261697769165,
      "logps/chosen": -13.690839767456055,
      "logps/rejected": -14.78709888458252,
      "loss": 0.0053,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -13.690839767456055,
      "rewards/margins": 1.0962581634521484,
      "rewards/rejected": -14.78709888458252,
      "step": 310
    },
    {
      "epoch": 0.16859006522829906,
      "grad_norm": 0.43788855250304975,
      "learning_rate": 5.614973262032086e-07,
      "logits/chosen": 2.143500804901123,
      "logits/rejected": 2.398965358734131,
      "logps/chosen": -16.440887451171875,
      "logps/rejected": -18.3090877532959,
      "loss": 0.0041,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -16.440887451171875,
      "rewards/margins": 1.8682035207748413,
      "rewards/rejected": -18.3090877532959,
      "step": 315
    },
    {
      "epoch": 0.1712660980097006,
      "grad_norm": 0.1515220476027461,
      "learning_rate": 5.70409982174688e-07,
      "logits/chosen": 2.4473185539245605,
      "logits/rejected": 2.6145071983337402,
      "logps/chosen": -17.95746421813965,
      "logps/rejected": -18.43484115600586,
      "loss": 0.0006,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -17.95746421813965,
      "rewards/margins": 0.4773777425289154,
      "rewards/rejected": -18.43484115600586,
      "step": 320
    },
    {
      "epoch": 0.17394213079110218,
      "grad_norm": 0.0934587649699255,
      "learning_rate": 5.793226381461676e-07,
      "logits/chosen": 3.1694912910461426,
      "logits/rejected": 3.363656997680664,
      "logps/chosen": -21.727306365966797,
      "logps/rejected": -23.835086822509766,
      "loss": 0.0012,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -21.727306365966797,
      "rewards/margins": 2.107780933380127,
      "rewards/rejected": -23.835086822509766,
      "step": 325
    },
    {
      "epoch": 0.17661816357250376,
      "grad_norm": 0.04289188015821034,
      "learning_rate": 5.88235294117647e-07,
      "logits/chosen": 3.4928061962127686,
      "logits/rejected": 3.451262950897217,
      "logps/chosen": -21.600542068481445,
      "logps/rejected": -24.803955078125,
      "loss": 0.0009,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -21.600542068481445,
      "rewards/margins": 3.203413486480713,
      "rewards/rejected": -24.803955078125,
      "step": 330
    },
    {
      "epoch": 0.17929419635390534,
      "grad_norm": 1.4645623935559486,
      "learning_rate": 5.971479500891266e-07,
      "logits/chosen": 4.0196733474731445,
      "logits/rejected": 4.058861255645752,
      "logps/chosen": -24.990421295166016,
      "logps/rejected": -25.708927154541016,
      "loss": 0.0007,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -24.990421295166016,
      "rewards/margins": 0.7185049653053284,
      "rewards/rejected": -25.708927154541016,
      "step": 335
    },
    {
      "epoch": 0.18197022913530692,
      "grad_norm": 0.04214895746629256,
      "learning_rate": 6.060606060606061e-07,
      "logits/chosen": 4.27630615234375,
      "logits/rejected": 4.299677848815918,
      "logps/chosen": -25.054115295410156,
      "logps/rejected": -27.55368423461914,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -25.054115295410156,
      "rewards/margins": 2.4995665550231934,
      "rewards/rejected": -27.55368423461914,
      "step": 340
    },
    {
      "epoch": 0.1846462619167085,
      "grad_norm": 0.04112378062024114,
      "learning_rate": 6.149732620320855e-07,
      "logits/chosen": 4.434001445770264,
      "logits/rejected": 4.55509090423584,
      "logps/chosen": -25.489980697631836,
      "logps/rejected": -26.87412452697754,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -25.489980697631836,
      "rewards/margins": 1.3841450214385986,
      "rewards/rejected": -26.87412452697754,
      "step": 345
    },
    {
      "epoch": 0.18732229469811004,
      "grad_norm": 0.00042743414594228086,
      "learning_rate": 6.238859180035651e-07,
      "logits/chosen": 4.505887031555176,
      "logits/rejected": 4.699957847595215,
      "logps/chosen": -25.814910888671875,
      "logps/rejected": -28.64125633239746,
      "loss": 0.001,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -25.814910888671875,
      "rewards/margins": 2.8263463973999023,
      "rewards/rejected": -28.64125633239746,
      "step": 350
    },
    {
      "epoch": 0.18999832747951162,
      "grad_norm": 0.035879226433014506,
      "learning_rate": 6.327985739750445e-07,
      "logits/chosen": 5.051446914672852,
      "logits/rejected": 4.953315734863281,
      "logps/chosen": -27.82820701599121,
      "logps/rejected": -29.5991268157959,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -27.82820701599121,
      "rewards/margins": 1.7709205150604248,
      "rewards/rejected": -29.5991268157959,
      "step": 355
    },
    {
      "epoch": 0.1926743602609132,
      "grad_norm": 0.0003175469168009553,
      "learning_rate": 6.417112299465241e-07,
      "logits/chosen": 5.0589118003845215,
      "logits/rejected": 5.075278282165527,
      "logps/chosen": -29.5541934967041,
      "logps/rejected": -30.4432430267334,
      "loss": 0.0002,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -29.5541934967041,
      "rewards/margins": 0.8890513181686401,
      "rewards/rejected": -30.4432430267334,
      "step": 360
    },
    {
      "epoch": 0.19535039304231477,
      "grad_norm": 0.0051673080346850316,
      "learning_rate": 6.506238859180035e-07,
      "logits/chosen": 4.929441928863525,
      "logits/rejected": 4.989992141723633,
      "logps/chosen": -28.256805419921875,
      "logps/rejected": -28.80402183532715,
      "loss": 0.0005,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -28.256805419921875,
      "rewards/margins": 0.5472115278244019,
      "rewards/rejected": -28.80402183532715,
      "step": 365
    },
    {
      "epoch": 0.19802642582371635,
      "grad_norm": 0.01031437573004315,
      "learning_rate": 6.59536541889483e-07,
      "logits/chosen": 5.048266410827637,
      "logits/rejected": 4.988327980041504,
      "logps/chosen": -27.683841705322266,
      "logps/rejected": -28.48501968383789,
      "loss": 0.0001,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -27.683841705322266,
      "rewards/margins": 0.801181435585022,
      "rewards/rejected": -28.48501968383789,
      "step": 370
    },
    {
      "epoch": 0.2007024586051179,
      "grad_norm": 0.04705777109675603,
      "learning_rate": 6.684491978609626e-07,
      "logits/chosen": 5.017124176025391,
      "logits/rejected": 4.993551731109619,
      "logps/chosen": -27.910104751586914,
      "logps/rejected": -30.440393447875977,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -27.910104751586914,
      "rewards/margins": 2.530287504196167,
      "rewards/rejected": -30.440393447875977,
      "step": 375
    },
    {
      "epoch": 0.20337849138651948,
      "grad_norm": 3.116720502400147e-05,
      "learning_rate": 6.77361853832442e-07,
      "logits/chosen": 5.353625297546387,
      "logits/rejected": 5.556002616882324,
      "logps/chosen": -29.577198028564453,
      "logps/rejected": -32.58041000366211,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -29.577198028564453,
      "rewards/margins": 3.003213405609131,
      "rewards/rejected": -32.58041000366211,
      "step": 380
    },
    {
      "epoch": 0.20605452416792105,
      "grad_norm": 0.00010713210858484258,
      "learning_rate": 6.862745098039216e-07,
      "logits/chosen": 5.450345516204834,
      "logits/rejected": 5.39117431640625,
      "logps/chosen": -29.253742218017578,
      "logps/rejected": -31.753101348876953,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -29.253742218017578,
      "rewards/margins": 2.4993581771850586,
      "rewards/rejected": -31.753101348876953,
      "step": 385
    },
    {
      "epoch": 0.20873055694932263,
      "grad_norm": 0.00553465724178475,
      "learning_rate": 6.95187165775401e-07,
      "logits/chosen": 5.020052433013916,
      "logits/rejected": 4.734940052032471,
      "logps/chosen": -29.95306968688965,
      "logps/rejected": -31.805065155029297,
      "loss": 0.0017,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -29.95306968688965,
      "rewards/margins": 1.8519980907440186,
      "rewards/rejected": -31.805065155029297,
      "step": 390
    },
    {
      "epoch": 0.2114065897307242,
      "grad_norm": 0.004926924435489586,
      "learning_rate": 7.040998217468806e-07,
      "logits/chosen": 5.617483615875244,
      "logits/rejected": 5.613303184509277,
      "logps/chosen": -29.747699737548828,
      "logps/rejected": -32.044166564941406,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -29.747699737548828,
      "rewards/margins": 2.296464204788208,
      "rewards/rejected": -32.044166564941406,
      "step": 395
    },
    {
      "epoch": 0.2140826225121258,
      "grad_norm": 0.10980056588215964,
      "learning_rate": 7.1301247771836e-07,
      "logits/chosen": 5.197115421295166,
      "logits/rejected": 5.246896266937256,
      "logps/chosen": -29.11849021911621,
      "logps/rejected": -31.373455047607422,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -29.11849021911621,
      "rewards/margins": 2.2549638748168945,
      "rewards/rejected": -31.373455047607422,
      "step": 400
    },
    {
      "epoch": 0.2140826225121258,
      "eval_logits/chosen": 4.508347988128662,
      "eval_logits/rejected": 4.525824546813965,
      "eval_logps/chosen": -29.474035263061523,
      "eval_logps/rejected": -31.223838806152344,
      "eval_loss": 0.00014844455290585756,
      "eval_rewards/accuracies": 0.5689911246299744,
      "eval_rewards/chosen": -29.474035263061523,
      "eval_rewards/margins": 1.749806523323059,
      "eval_rewards/rejected": -31.223838806152344,
      "eval_runtime": 41.4755,
      "eval_samples_per_second": 32.429,
      "eval_steps_per_second": 8.125,
      "step": 400
    },
    {
      "epoch": 0.21675865529352734,
      "grad_norm": 4.0460074400129147e-07,
      "learning_rate": 7.219251336898395e-07,
      "logits/chosen": 5.426613807678223,
      "logits/rejected": 5.434569358825684,
      "logps/chosen": -31.552480697631836,
      "logps/rejected": -33.021236419677734,
      "loss": 0.0001,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -31.552480697631836,
      "rewards/margins": 1.468758225440979,
      "rewards/rejected": -33.021236419677734,
      "step": 405
    },
    {
      "epoch": 0.2194346880749289,
      "grad_norm": 0.0018394903367872774,
      "learning_rate": 7.30837789661319e-07,
      "logits/chosen": 5.585395812988281,
      "logits/rejected": 5.505690574645996,
      "logps/chosen": -30.765277862548828,
      "logps/rejected": -33.54123306274414,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -30.765277862548828,
      "rewards/margins": 2.7759573459625244,
      "rewards/rejected": -33.54123306274414,
      "step": 410
    },
    {
      "epoch": 0.2221107208563305,
      "grad_norm": 9.519517465463827e-07,
      "learning_rate": 7.397504456327985e-07,
      "logits/chosen": 5.678066730499268,
      "logits/rejected": 5.77048397064209,
      "logps/chosen": -32.274635314941406,
      "logps/rejected": -33.142452239990234,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -32.274635314941406,
      "rewards/margins": 0.8678176999092102,
      "rewards/rejected": -33.142452239990234,
      "step": 415
    },
    {
      "epoch": 0.22478675363773207,
      "grad_norm": 0.028954290644148215,
      "learning_rate": 7.486631016042781e-07,
      "logits/chosen": 5.4329633712768555,
      "logits/rejected": 5.234847068786621,
      "logps/chosen": -30.790943145751953,
      "logps/rejected": -33.33657455444336,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -30.790943145751953,
      "rewards/margins": 2.545635938644409,
      "rewards/rejected": -33.33657455444336,
      "step": 420
    },
    {
      "epoch": 0.22746278641913364,
      "grad_norm": 0.0006987982490430461,
      "learning_rate": 7.575757575757575e-07,
      "logits/chosen": 5.261269569396973,
      "logits/rejected": 5.335232257843018,
      "logps/chosen": -30.87042808532715,
      "logps/rejected": -33.61421203613281,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -30.87042808532715,
      "rewards/margins": 2.743783950805664,
      "rewards/rejected": -33.61421203613281,
      "step": 425
    },
    {
      "epoch": 0.2301388192005352,
      "grad_norm": 0.008281334251306848,
      "learning_rate": 7.664884135472371e-07,
      "logits/chosen": 5.641324043273926,
      "logits/rejected": 5.514430522918701,
      "logps/chosen": -28.550350189208984,
      "logps/rejected": -32.46430587768555,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -28.550350189208984,
      "rewards/margins": 3.9139564037323,
      "rewards/rejected": -32.46430587768555,
      "step": 430
    },
    {
      "epoch": 0.23281485198193677,
      "grad_norm": 0.0001335339701865778,
      "learning_rate": 7.754010695187165e-07,
      "logits/chosen": 5.681117057800293,
      "logits/rejected": 5.728180408477783,
      "logps/chosen": -30.91018295288086,
      "logps/rejected": -31.73850440979004,
      "loss": 0.0001,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -30.91018295288086,
      "rewards/margins": 0.8283231854438782,
      "rewards/rejected": -31.73850440979004,
      "step": 435
    },
    {
      "epoch": 0.23549088476333835,
      "grad_norm": 0.0003905120367096067,
      "learning_rate": 7.84313725490196e-07,
      "logits/chosen": 5.308198928833008,
      "logits/rejected": 5.242469310760498,
      "logps/chosen": -28.735559463500977,
      "logps/rejected": -30.480484008789062,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -28.735559463500977,
      "rewards/margins": 1.7449262142181396,
      "rewards/rejected": -30.480484008789062,
      "step": 440
    },
    {
      "epoch": 0.23816691754473993,
      "grad_norm": 0.035724152997041105,
      "learning_rate": 7.932263814616755e-07,
      "logits/chosen": 5.429460048675537,
      "logits/rejected": 5.417057037353516,
      "logps/chosen": -29.862049102783203,
      "logps/rejected": -31.693466186523438,
      "loss": 0.0001,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -29.862049102783203,
      "rewards/margins": 1.8314167261123657,
      "rewards/rejected": -31.693466186523438,
      "step": 445
    },
    {
      "epoch": 0.2408429503261415,
      "grad_norm": 0.0008963971978067854,
      "learning_rate": 8.02139037433155e-07,
      "logits/chosen": 5.698996543884277,
      "logits/rejected": 5.760076522827148,
      "logps/chosen": -31.261394500732422,
      "logps/rejected": -33.20148468017578,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -31.261394500732422,
      "rewards/margins": 1.9400899410247803,
      "rewards/rejected": -33.20148468017578,
      "step": 450
    },
    {
      "epoch": 0.24351898310754308,
      "grad_norm": 0.0002816622127148177,
      "learning_rate": 8.110516934046346e-07,
      "logits/chosen": 5.82552433013916,
      "logits/rejected": 5.86575174331665,
      "logps/chosen": -29.802892684936523,
      "logps/rejected": -31.987655639648438,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -29.802892684936523,
      "rewards/margins": 2.184760570526123,
      "rewards/rejected": -31.987655639648438,
      "step": 455
    },
    {
      "epoch": 0.24619501588894463,
      "grad_norm": 0.0007354757513253456,
      "learning_rate": 8.19964349376114e-07,
      "logits/chosen": 5.799552917480469,
      "logits/rejected": 5.78856897354126,
      "logps/chosen": -30.004852294921875,
      "logps/rejected": -32.112491607666016,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -30.004852294921875,
      "rewards/margins": 2.1076369285583496,
      "rewards/rejected": -32.112491607666016,
      "step": 460
    },
    {
      "epoch": 0.2488710486703462,
      "grad_norm": 0.00249200164181285,
      "learning_rate": 8.288770053475936e-07,
      "logits/chosen": 5.848760604858398,
      "logits/rejected": 5.7724995613098145,
      "logps/chosen": -32.0567512512207,
      "logps/rejected": -33.11432647705078,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -32.0567512512207,
      "rewards/margins": 1.057578444480896,
      "rewards/rejected": -33.11432647705078,
      "step": 465
    },
    {
      "epoch": 0.2515470814517478,
      "grad_norm": 0.008788564272295995,
      "learning_rate": 8.37789661319073e-07,
      "logits/chosen": 6.024893760681152,
      "logits/rejected": 6.152130126953125,
      "logps/chosen": -33.38634490966797,
      "logps/rejected": -33.031341552734375,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -33.38634490966797,
      "rewards/margins": -0.3550027012825012,
      "rewards/rejected": -33.031341552734375,
      "step": 470
    },
    {
      "epoch": 0.25422311423314936,
      "grad_norm": 0.008045023749322238,
      "learning_rate": 8.467023172905525e-07,
      "logits/chosen": 6.149905204772949,
      "logits/rejected": 6.1068315505981445,
      "logps/chosen": -31.69088363647461,
      "logps/rejected": -35.666839599609375,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -31.69088363647461,
      "rewards/margins": 3.9759507179260254,
      "rewards/rejected": -35.666839599609375,
      "step": 475
    },
    {
      "epoch": 0.2568991470145509,
      "grad_norm": 0.24161562793150843,
      "learning_rate": 8.55614973262032e-07,
      "logits/chosen": 6.131672382354736,
      "logits/rejected": 6.029187202453613,
      "logps/chosen": -31.79244041442871,
      "logps/rejected": -34.317020416259766,
      "loss": 0.0001,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -31.79244041442871,
      "rewards/margins": 2.5245766639709473,
      "rewards/rejected": -34.317020416259766,
      "step": 480
    },
    {
      "epoch": 0.2595751797959525,
      "grad_norm": 0.0006102102592899613,
      "learning_rate": 8.645276292335115e-07,
      "logits/chosen": 6.5127692222595215,
      "logits/rejected": 6.600030422210693,
      "logps/chosen": -33.49677276611328,
      "logps/rejected": -35.524227142333984,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -33.49677276611328,
      "rewards/margins": 2.027453899383545,
      "rewards/rejected": -35.524227142333984,
      "step": 485
    },
    {
      "epoch": 0.26225121257735406,
      "grad_norm": 0.0009493708808510832,
      "learning_rate": 8.734402852049911e-07,
      "logits/chosen": 5.9582061767578125,
      "logits/rejected": 6.046202182769775,
      "logps/chosen": -33.23215866088867,
      "logps/rejected": -34.372230529785156,
      "loss": 0.0002,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -33.23215866088867,
      "rewards/margins": 1.140074610710144,
      "rewards/rejected": -34.372230529785156,
      "step": 490
    },
    {
      "epoch": 0.26492724535875567,
      "grad_norm": 0.00022864888927709976,
      "learning_rate": 8.823529411764705e-07,
      "logits/chosen": 6.260375022888184,
      "logits/rejected": 6.266873836517334,
      "logps/chosen": -35.20757293701172,
      "logps/rejected": -34.25310134887695,
      "loss": 0.0,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -35.20757293701172,
      "rewards/margins": -0.9544719457626343,
      "rewards/rejected": -34.25310134887695,
      "step": 495
    },
    {
      "epoch": 0.2676032781401572,
      "grad_norm": 0.00019236693309136163,
      "learning_rate": 8.912655971479501e-07,
      "logits/chosen": 6.197518348693848,
      "logits/rejected": 6.117464065551758,
      "logps/chosen": -33.92963790893555,
      "logps/rejected": -34.88914108276367,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -33.92963790893555,
      "rewards/margins": 0.9595043063163757,
      "rewards/rejected": -34.88914108276367,
      "step": 500
    },
    {
      "epoch": 0.27027931092155877,
      "grad_norm": 0.11369246832499683,
      "learning_rate": 9.001782531194295e-07,
      "logits/chosen": 6.148838043212891,
      "logits/rejected": 6.055180549621582,
      "logps/chosen": -32.2354850769043,
      "logps/rejected": -33.38620376586914,
      "loss": 0.0014,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -32.2354850769043,
      "rewards/margins": 1.1507171392440796,
      "rewards/rejected": -33.38620376586914,
      "step": 505
    },
    {
      "epoch": 0.2729553437029604,
      "grad_norm": 0.00024122807731692076,
      "learning_rate": 9.09090909090909e-07,
      "logits/chosen": 6.060546875,
      "logits/rejected": 6.003035068511963,
      "logps/chosen": -33.80747985839844,
      "logps/rejected": -34.501033782958984,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -33.80747985839844,
      "rewards/margins": 0.6935546398162842,
      "rewards/rejected": -34.501033782958984,
      "step": 510
    },
    {
      "epoch": 0.2756313764843619,
      "grad_norm": 0.0017872707280551482,
      "learning_rate": 9.180035650623885e-07,
      "logits/chosen": 5.7357988357543945,
      "logits/rejected": 5.852870941162109,
      "logps/chosen": -31.31736183166504,
      "logps/rejected": -33.037017822265625,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -31.31736183166504,
      "rewards/margins": 1.7196519374847412,
      "rewards/rejected": -33.037017822265625,
      "step": 515
    },
    {
      "epoch": 0.27830740926576353,
      "grad_norm": 2.6238419454377426e-05,
      "learning_rate": 9.26916221033868e-07,
      "logits/chosen": 6.473957061767578,
      "logits/rejected": 6.2191596031188965,
      "logps/chosen": -34.885589599609375,
      "logps/rejected": -35.81165313720703,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -34.885589599609375,
      "rewards/margins": 0.9260598421096802,
      "rewards/rejected": -35.81165313720703,
      "step": 520
    },
    {
      "epoch": 0.2809834420471651,
      "grad_norm": 0.0014621432044564487,
      "learning_rate": 9.358288770053476e-07,
      "logits/chosen": 6.131948947906494,
      "logits/rejected": 6.05927038192749,
      "logps/chosen": -33.809661865234375,
      "logps/rejected": -35.10086441040039,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -33.809661865234375,
      "rewards/margins": 1.2912040948867798,
      "rewards/rejected": -35.10086441040039,
      "step": 525
    },
    {
      "epoch": 0.2836594748285666,
      "grad_norm": 0.035805550797865744,
      "learning_rate": 9.44741532976827e-07,
      "logits/chosen": 6.354075908660889,
      "logits/rejected": 6.551912784576416,
      "logps/chosen": -33.002723693847656,
      "logps/rejected": -34.30055618286133,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -33.002723693847656,
      "rewards/margins": 1.2978332042694092,
      "rewards/rejected": -34.30055618286133,
      "step": 530
    },
    {
      "epoch": 0.28633550760996823,
      "grad_norm": 1.2210176107842878e-05,
      "learning_rate": 9.536541889483066e-07,
      "logits/chosen": 6.335618019104004,
      "logits/rejected": 5.982363700866699,
      "logps/chosen": -32.45991134643555,
      "logps/rejected": -36.38939666748047,
      "loss": 0.0,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -32.45991134643555,
      "rewards/margins": 3.929482936859131,
      "rewards/rejected": -36.38939666748047,
      "step": 535
    },
    {
      "epoch": 0.2890115403913698,
      "grad_norm": 0.0014578279830980442,
      "learning_rate": 9.62566844919786e-07,
      "logits/chosen": 6.4163498878479,
      "logits/rejected": 6.378708362579346,
      "logps/chosen": -32.91603469848633,
      "logps/rejected": -34.10979461669922,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -32.91603469848633,
      "rewards/margins": 1.193760871887207,
      "rewards/rejected": -34.10979461669922,
      "step": 540
    },
    {
      "epoch": 0.2916875731727714,
      "grad_norm": 0.009891179319808723,
      "learning_rate": 9.714795008912655e-07,
      "logits/chosen": 6.257334232330322,
      "logits/rejected": 6.111904144287109,
      "logps/chosen": -32.97081756591797,
      "logps/rejected": -36.07368087768555,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -32.97081756591797,
      "rewards/margins": 3.102867841720581,
      "rewards/rejected": -36.07368087768555,
      "step": 545
    },
    {
      "epoch": 0.29436360595417294,
      "grad_norm": 0.00010441327591347197,
      "learning_rate": 9.80392156862745e-07,
      "logits/chosen": 6.489959716796875,
      "logits/rejected": 6.449127197265625,
      "logps/chosen": -33.431461334228516,
      "logps/rejected": -34.821311950683594,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -33.431461334228516,
      "rewards/margins": 1.389845371246338,
      "rewards/rejected": -34.821311950683594,
      "step": 550
    },
    {
      "epoch": 0.2970396387355745,
      "grad_norm": 0.006376648448372838,
      "learning_rate": 9.893048128342244e-07,
      "logits/chosen": 6.40233850479126,
      "logits/rejected": 6.086165904998779,
      "logps/chosen": -34.14889907836914,
      "logps/rejected": -34.164825439453125,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -34.14889907836914,
      "rewards/margins": 0.01592273637652397,
      "rewards/rejected": -34.164825439453125,
      "step": 555
    },
    {
      "epoch": 0.2997156715169761,
      "grad_norm": 0.017814151085457603,
      "learning_rate": 9.98217468805704e-07,
      "logits/chosen": 6.360291957855225,
      "logits/rejected": 6.425871849060059,
      "logps/chosen": -33.9993896484375,
      "logps/rejected": -34.31085968017578,
      "loss": 0.0,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -33.9993896484375,
      "rewards/margins": 0.3114696443080902,
      "rewards/rejected": -34.31085968017578,
      "step": 560
    },
    {
      "epoch": 0.30239170429837764,
      "grad_norm": 0.0005208945254706291,
      "learning_rate": 9.999984476788462e-07,
      "logits/chosen": 6.604605197906494,
      "logits/rejected": 6.542850494384766,
      "logps/chosen": -33.41400909423828,
      "logps/rejected": -35.229984283447266,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -33.41400909423828,
      "rewards/margins": 1.8159730434417725,
      "rewards/rejected": -35.229984283447266,
      "step": 565
    },
    {
      "epoch": 0.30506773707977924,
      "grad_norm": 2.1017687150351856e-05,
      "learning_rate": 9.999921413906797e-07,
      "logits/chosen": 6.328225135803223,
      "logits/rejected": 6.258444786071777,
      "logps/chosen": -32.51350784301758,
      "logps/rejected": -35.06706237792969,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -32.51350784301758,
      "rewards/margins": 2.553555488586426,
      "rewards/rejected": -35.06706237792969,
      "step": 570
    },
    {
      "epoch": 0.3077437698611808,
      "grad_norm": 3.325106964892115e-06,
      "learning_rate": 9.999809841765644e-07,
      "logits/chosen": 6.846964359283447,
      "logits/rejected": 7.237646579742432,
      "logps/chosen": -34.89623260498047,
      "logps/rejected": -36.382240295410156,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -34.89623260498047,
      "rewards/margins": 1.4860029220581055,
      "rewards/rejected": -36.382240295410156,
      "step": 575
    },
    {
      "epoch": 0.3104198026425824,
      "grad_norm": 9.052197419586339e-06,
      "learning_rate": 9.999649761447477e-07,
      "logits/chosen": 7.036424160003662,
      "logits/rejected": 6.716401100158691,
      "logps/chosen": -34.93789291381836,
      "logps/rejected": -36.21113204956055,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -34.93789291381836,
      "rewards/margins": 1.273238182067871,
      "rewards/rejected": -36.21113204956055,
      "step": 580
    },
    {
      "epoch": 0.31309583542398395,
      "grad_norm": 0.0013050613006307422,
      "learning_rate": 9.999441174505398e-07,
      "logits/chosen": 6.6325883865356445,
      "logits/rejected": 6.768698215484619,
      "logps/chosen": -34.55119323730469,
      "logps/rejected": -36.57561492919922,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -34.55119323730469,
      "rewards/margins": 2.0244240760803223,
      "rewards/rejected": -36.57561492919922,
      "step": 585
    },
    {
      "epoch": 0.3157718682053855,
      "grad_norm": 4.135330629953316e-07,
      "learning_rate": 9.999184082963116e-07,
      "logits/chosen": 6.797358512878418,
      "logits/rejected": 6.864867210388184,
      "logps/chosen": -34.69548034667969,
      "logps/rejected": -36.387203216552734,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -34.69548034667969,
      "rewards/margins": 1.6917203664779663,
      "rewards/rejected": -36.387203216552734,
      "step": 590
    },
    {
      "epoch": 0.3184479009867871,
      "grad_norm": 0.0011523503667480658,
      "learning_rate": 9.998878489314937e-07,
      "logits/chosen": 6.9756598472595215,
      "logits/rejected": 6.642075538635254,
      "logps/chosen": -34.88782501220703,
      "logps/rejected": -37.1215705871582,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -34.88782501220703,
      "rewards/margins": 2.2337491512298584,
      "rewards/rejected": -37.1215705871582,
      "step": 595
    },
    {
      "epoch": 0.32112393376818865,
      "grad_norm": 0.0008748957514337547,
      "learning_rate": 9.99852439652573e-07,
      "logits/chosen": 6.799266815185547,
      "logits/rejected": 6.709418296813965,
      "logps/chosen": -34.214942932128906,
      "logps/rejected": -35.7734260559082,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -34.214942932128906,
      "rewards/margins": 1.5584805011749268,
      "rewards/rejected": -35.7734260559082,
      "step": 600
    },
    {
      "epoch": 0.32379996654959026,
      "grad_norm": 4.320098313797268e-06,
      "learning_rate": 9.998121808030904e-07,
      "logits/chosen": 6.866464138031006,
      "logits/rejected": 6.8782806396484375,
      "logps/chosen": -35.04458999633789,
      "logps/rejected": -36.22504806518555,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -35.04458999633789,
      "rewards/margins": 1.1804614067077637,
      "rewards/rejected": -36.22504806518555,
      "step": 605
    },
    {
      "epoch": 0.3264759993309918,
      "grad_norm": 0.0014950732437660522,
      "learning_rate": 9.997670727736379e-07,
      "logits/chosen": 6.629367828369141,
      "logits/rejected": 6.501309394836426,
      "logps/chosen": -34.2564582824707,
      "logps/rejected": -36.680213928222656,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -34.2564582824707,
      "rewards/margins": 2.4237563610076904,
      "rewards/rejected": -36.680213928222656,
      "step": 610
    },
    {
      "epoch": 0.32915203211239336,
      "grad_norm": 4.893172143427414e-05,
      "learning_rate": 9.99717116001853e-07,
      "logits/chosen": 6.801131248474121,
      "logits/rejected": 6.845280647277832,
      "logps/chosen": -33.56496047973633,
      "logps/rejected": -36.0307731628418,
      "loss": 0.0001,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -33.56496047973633,
      "rewards/margins": 2.465816020965576,
      "rewards/rejected": -36.0307731628418,
      "step": 615
    },
    {
      "epoch": 0.33182806489379496,
      "grad_norm": 6.719738160788306e-07,
      "learning_rate": 9.996623109724173e-07,
      "logits/chosen": 6.665761470794678,
      "logits/rejected": 6.588541507720947,
      "logps/chosen": -34.28596878051758,
      "logps/rejected": -35.750144958496094,
      "loss": 0.0001,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -34.28596878051758,
      "rewards/margins": 1.4641757011413574,
      "rewards/rejected": -35.750144958496094,
      "step": 620
    },
    {
      "epoch": 0.3345040976751965,
      "grad_norm": 1.4929370534075512e-06,
      "learning_rate": 9.996026582170488e-07,
      "logits/chosen": 6.752054691314697,
      "logits/rejected": 6.4877800941467285,
      "logps/chosen": -34.279685974121094,
      "logps/rejected": -36.55657196044922,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -34.279685974121094,
      "rewards/margins": 2.276893138885498,
      "rewards/rejected": -36.55657196044922,
      "step": 625
    },
    {
      "epoch": 0.3371801304565981,
      "grad_norm": 4.105902258468344e-06,
      "learning_rate": 9.995381583144996e-07,
      "logits/chosen": 6.7286577224731445,
      "logits/rejected": 6.735819339752197,
      "logps/chosen": -33.57708740234375,
      "logps/rejected": -35.56078338623047,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -33.57708740234375,
      "rewards/margins": 1.9836965799331665,
      "rewards/rejected": -35.56078338623047,
      "step": 630
    },
    {
      "epoch": 0.33985616323799966,
      "grad_norm": 0.0001693453599894774,
      "learning_rate": 9.994688118905471e-07,
      "logits/chosen": 7.191317558288574,
      "logits/rejected": 6.851290225982666,
      "logps/chosen": -34.74304962158203,
      "logps/rejected": -37.620323181152344,
      "loss": 0.0,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -34.74304962158203,
      "rewards/margins": 2.877272367477417,
      "rewards/rejected": -37.620323181152344,
      "step": 635
    },
    {
      "epoch": 0.3425321960194012,
      "grad_norm": 6.510103094074435e-05,
      "learning_rate": 9.993946196179912e-07,
      "logits/chosen": 6.8470869064331055,
      "logits/rejected": 6.5663275718688965,
      "logps/chosen": -35.054527282714844,
      "logps/rejected": -36.634910583496094,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -35.054527282714844,
      "rewards/margins": 1.5803860425949097,
      "rewards/rejected": -36.634910583496094,
      "step": 640
    },
    {
      "epoch": 0.3452082288008028,
      "grad_norm": 7.303451703777848e-06,
      "learning_rate": 9.993155822166455e-07,
      "logits/chosen": 7.290508270263672,
      "logits/rejected": 7.090371131896973,
      "logps/chosen": -35.157752990722656,
      "logps/rejected": -36.928775787353516,
      "loss": 0.0001,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -35.157752990722656,
      "rewards/margins": 1.77102530002594,
      "rewards/rejected": -36.928775787353516,
      "step": 645
    },
    {
      "epoch": 0.34788426158220437,
      "grad_norm": 1.0119487349165119e-05,
      "learning_rate": 9.992317004533313e-07,
      "logits/chosen": 7.054312229156494,
      "logits/rejected": 7.178700923919678,
      "logps/chosen": -34.85392379760742,
      "logps/rejected": -36.34272003173828,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -34.85392379760742,
      "rewards/margins": 1.4887895584106445,
      "rewards/rejected": -36.34272003173828,
      "step": 650
    },
    {
      "epoch": 0.350560294363606,
      "grad_norm": 0.00014489568910877493,
      "learning_rate": 9.991429751418696e-07,
      "logits/chosen": 6.902565956115723,
      "logits/rejected": 7.105119228363037,
      "logps/chosen": -35.772254943847656,
      "logps/rejected": -36.574886322021484,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -35.772254943847656,
      "rewards/margins": 0.8026283383369446,
      "rewards/rejected": -36.574886322021484,
      "step": 655
    },
    {
      "epoch": 0.3532363271450075,
      "grad_norm": 0.011996029600377949,
      "learning_rate": 9.99049407143074e-07,
      "logits/chosen": 7.235396385192871,
      "logits/rejected": 6.983212471008301,
      "logps/chosen": -35.8619499206543,
      "logps/rejected": -36.84368133544922,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -35.8619499206543,
      "rewards/margins": 0.981733500957489,
      "rewards/rejected": -36.84368133544922,
      "step": 660
    },
    {
      "epoch": 0.35591235992640907,
      "grad_norm": 4.19207999024116e-07,
      "learning_rate": 9.989509973647416e-07,
      "logits/chosen": 6.924723148345947,
      "logits/rejected": 6.78842830657959,
      "logps/chosen": -35.5040283203125,
      "logps/rejected": -36.55635070800781,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -35.5040283203125,
      "rewards/margins": 1.0523169040679932,
      "rewards/rejected": -36.55635070800781,
      "step": 665
    },
    {
      "epoch": 0.3585883927078107,
      "grad_norm": 3.938915632090982e-05,
      "learning_rate": 9.988477467616445e-07,
      "logits/chosen": 7.461147308349609,
      "logits/rejected": 7.200994968414307,
      "logps/chosen": -36.98133850097656,
      "logps/rejected": -39.15379333496094,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -36.98133850097656,
      "rewards/margins": 2.172454357147217,
      "rewards/rejected": -39.15379333496094,
      "step": 670
    },
    {
      "epoch": 0.3612644254892122,
      "grad_norm": 0.009116824378483615,
      "learning_rate": 9.987396563355205e-07,
      "logits/chosen": 7.330562591552734,
      "logits/rejected": 7.280575752258301,
      "logps/chosen": -36.07804870605469,
      "logps/rejected": -37.525672912597656,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -36.07804870605469,
      "rewards/margins": 1.4476271867752075,
      "rewards/rejected": -37.525672912597656,
      "step": 675
    },
    {
      "epoch": 0.36394045827061383,
      "grad_norm": 0.0031837750399864926,
      "learning_rate": 9.986267271350631e-07,
      "logits/chosen": 7.121218681335449,
      "logits/rejected": 6.910025596618652,
      "logps/chosen": -35.7744026184082,
      "logps/rejected": -37.34346389770508,
      "loss": 0.0009,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -35.7744026184082,
      "rewards/margins": 1.5690600872039795,
      "rewards/rejected": -37.34346389770508,
      "step": 680
    },
    {
      "epoch": 0.3666164910520154,
      "grad_norm": 3.116779140640639e-06,
      "learning_rate": 9.985089602559123e-07,
      "logits/chosen": 7.288196563720703,
      "logits/rejected": 6.879519462585449,
      "logps/chosen": -34.97138214111328,
      "logps/rejected": -37.335426330566406,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -34.97138214111328,
      "rewards/margins": 2.364042282104492,
      "rewards/rejected": -37.335426330566406,
      "step": 685
    },
    {
      "epoch": 0.369292523833417,
      "grad_norm": 0.00010529115571167023,
      "learning_rate": 9.983863568406428e-07,
      "logits/chosen": 7.017849922180176,
      "logits/rejected": 7.060793399810791,
      "logps/chosen": -35.998897552490234,
      "logps/rejected": -36.42427444458008,
      "loss": 0.0003,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -35.998897552490234,
      "rewards/margins": 0.4253758490085602,
      "rewards/rejected": -36.42427444458008,
      "step": 690
    },
    {
      "epoch": 0.37196855661481854,
      "grad_norm": 0.002090983438652169,
      "learning_rate": 9.982589180787532e-07,
      "logits/chosen": 7.2100653648376465,
      "logits/rejected": 7.025811672210693,
      "logps/chosen": -36.97612380981445,
      "logps/rejected": -37.301292419433594,
      "loss": 0.0,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -36.97612380981445,
      "rewards/margins": 0.32516971230506897,
      "rewards/rejected": -37.301292419433594,
      "step": 695
    },
    {
      "epoch": 0.3746445893962201,
      "grad_norm": 2.97456526812461e-05,
      "learning_rate": 9.981266452066553e-07,
      "logits/chosen": 7.534338474273682,
      "logits/rejected": 7.202698707580566,
      "logps/chosen": -36.010597229003906,
      "logps/rejected": -37.713661193847656,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -36.010597229003906,
      "rewards/margins": 1.7030658721923828,
      "rewards/rejected": -37.713661193847656,
      "step": 700
    },
    {
      "epoch": 0.3773206221776217,
      "grad_norm": 0.048827650815992704,
      "learning_rate": 9.979895395076608e-07,
      "logits/chosen": 7.625168800354004,
      "logits/rejected": 7.130354404449463,
      "logps/chosen": -37.0987434387207,
      "logps/rejected": -38.473106384277344,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -37.0987434387207,
      "rewards/margins": 1.3743607997894287,
      "rewards/rejected": -38.473106384277344,
      "step": 705
    },
    {
      "epoch": 0.37999665495902324,
      "grad_norm": 2.5723916928794195e-05,
      "learning_rate": 9.9784760231197e-07,
      "logits/chosen": 7.514035224914551,
      "logits/rejected": 7.577850341796875,
      "logps/chosen": -37.40643310546875,
      "logps/rejected": -39.87279510498047,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -37.40643310546875,
      "rewards/margins": 2.466355800628662,
      "rewards/rejected": -39.87279510498047,
      "step": 710
    },
    {
      "epoch": 0.38267268774042484,
      "grad_norm": 0.02494130431079218,
      "learning_rate": 9.97700834996658e-07,
      "logits/chosen": 7.218267917633057,
      "logits/rejected": 6.9841156005859375,
      "logps/chosen": -36.930870056152344,
      "logps/rejected": -38.65314865112305,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -36.930870056152344,
      "rewards/margins": 1.7222788333892822,
      "rewards/rejected": -38.65314865112305,
      "step": 715
    },
    {
      "epoch": 0.3853487205218264,
      "grad_norm": 2.227201395744357e-05,
      "learning_rate": 9.97549238985662e-07,
      "logits/chosen": 7.365689754486084,
      "logits/rejected": 6.939404487609863,
      "logps/chosen": -36.51023483276367,
      "logps/rejected": -38.6107292175293,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -36.51023483276367,
      "rewards/margins": 2.100494861602783,
      "rewards/rejected": -38.6107292175293,
      "step": 720
    },
    {
      "epoch": 0.38802475330322794,
      "grad_norm": 1.113062910748177e-07,
      "learning_rate": 9.973928157497674e-07,
      "logits/chosen": 7.783537864685059,
      "logits/rejected": 7.40472936630249,
      "logps/chosen": -37.044673919677734,
      "logps/rejected": -38.82173156738281,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -37.044673919677734,
      "rewards/margins": 1.7770541906356812,
      "rewards/rejected": -38.82173156738281,
      "step": 725
    },
    {
      "epoch": 0.39070078608462955,
      "grad_norm": 0.00027254899155386783,
      "learning_rate": 9.972315668065927e-07,
      "logits/chosen": 7.683008670806885,
      "logits/rejected": 7.494215965270996,
      "logps/chosen": -38.76002502441406,
      "logps/rejected": -39.1976203918457,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -38.76002502441406,
      "rewards/margins": 0.4375886023044586,
      "rewards/rejected": -39.1976203918457,
      "step": 730
    },
    {
      "epoch": 0.3933768188660311,
      "grad_norm": 0.0019263658636157074,
      "learning_rate": 9.97065493720576e-07,
      "logits/chosen": 7.4932732582092285,
      "logits/rejected": 7.3577423095703125,
      "logps/chosen": -35.77894973754883,
      "logps/rejected": -36.88482666015625,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -35.77894973754883,
      "rewards/margins": 1.105878233909607,
      "rewards/rejected": -36.88482666015625,
      "step": 735
    },
    {
      "epoch": 0.3960528516474327,
      "grad_norm": 8.810174202265158e-07,
      "learning_rate": 9.968945981029594e-07,
      "logits/chosen": 7.506727695465088,
      "logits/rejected": 7.288453102111816,
      "logps/chosen": -35.85859298706055,
      "logps/rejected": -38.346832275390625,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -35.85859298706055,
      "rewards/margins": 2.488232135772705,
      "rewards/rejected": -38.346832275390625,
      "step": 740
    },
    {
      "epoch": 0.39872888442883425,
      "grad_norm": 1.3527867281810915e-05,
      "learning_rate": 9.967188816117726e-07,
      "logits/chosen": 7.556031227111816,
      "logits/rejected": 7.525795936584473,
      "logps/chosen": -36.91010284423828,
      "logps/rejected": -40.40468978881836,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -36.91010284423828,
      "rewards/margins": 3.4945881366729736,
      "rewards/rejected": -40.40468978881836,
      "step": 745
    },
    {
      "epoch": 0.4014049172102358,
      "grad_norm": 0.00016034383285586597,
      "learning_rate": 9.965383459518179e-07,
      "logits/chosen": 7.278078556060791,
      "logits/rejected": 7.1918463706970215,
      "logps/chosen": -35.90861129760742,
      "logps/rejected": -38.47333526611328,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -35.90861129760742,
      "rewards/margins": 2.564725875854492,
      "rewards/rejected": -38.47333526611328,
      "step": 750
    },
    {
      "epoch": 0.4040809499916374,
      "grad_norm": 2.2081192706073446e-07,
      "learning_rate": 9.963529928746533e-07,
      "logits/chosen": 7.5855560302734375,
      "logits/rejected": 7.1510820388793945,
      "logps/chosen": -35.81825637817383,
      "logps/rejected": -38.087852478027344,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -35.81825637817383,
      "rewards/margins": 2.269597291946411,
      "rewards/rejected": -38.087852478027344,
      "step": 755
    },
    {
      "epoch": 0.40675698277303896,
      "grad_norm": 0.0012603887909422699,
      "learning_rate": 9.961628241785746e-07,
      "logits/chosen": 7.386877536773682,
      "logits/rejected": 7.278040885925293,
      "logps/chosen": -36.343971252441406,
      "logps/rejected": -37.227210998535156,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -36.343971252441406,
      "rewards/margins": 0.8832328915596008,
      "rewards/rejected": -37.227210998535156,
      "step": 760
    },
    {
      "epoch": 0.40943301555444056,
      "grad_norm": 0.0022033366806607176,
      "learning_rate": 9.959678417085998e-07,
      "logits/chosen": 7.282350063323975,
      "logits/rejected": 7.231588840484619,
      "logps/chosen": -36.273895263671875,
      "logps/rejected": -37.92772674560547,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -36.273895263671875,
      "rewards/margins": 1.6538302898406982,
      "rewards/rejected": -37.92772674560547,
      "step": 765
    },
    {
      "epoch": 0.4121090483358421,
      "grad_norm": 3.074897623286504e-07,
      "learning_rate": 9.957680473564493e-07,
      "logits/chosen": 7.903887748718262,
      "logits/rejected": 7.689046382904053,
      "logps/chosen": -38.436553955078125,
      "logps/rejected": -41.07474136352539,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -38.436553955078125,
      "rewards/margins": 2.6381802558898926,
      "rewards/rejected": -41.07474136352539,
      "step": 770
    },
    {
      "epoch": 0.41478508111724366,
      "grad_norm": 0.0040569328807811625,
      "learning_rate": 9.95563443060529e-07,
      "logits/chosen": 7.836782932281494,
      "logits/rejected": 7.382288932800293,
      "logps/chosen": -36.24165344238281,
      "logps/rejected": -38.431461334228516,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -36.24165344238281,
      "rewards/margins": 2.1898043155670166,
      "rewards/rejected": -38.431461334228516,
      "step": 775
    },
    {
      "epoch": 0.41746111389864526,
      "grad_norm": 1.2960539710483852e-05,
      "learning_rate": 9.95354030805911e-07,
      "logits/chosen": 7.578127861022949,
      "logits/rejected": 7.2394304275512695,
      "logps/chosen": -38.148136138916016,
      "logps/rejected": -39.74500274658203,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -38.148136138916016,
      "rewards/margins": 1.596866488456726,
      "rewards/rejected": -39.74500274658203,
      "step": 780
    },
    {
      "epoch": 0.4201371466800468,
      "grad_norm": 0.0002551349932661308,
      "learning_rate": 9.951398126243133e-07,
      "logits/chosen": 7.510928153991699,
      "logits/rejected": 7.400703430175781,
      "logps/chosen": -37.220542907714844,
      "logps/rejected": -38.346168518066406,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -37.220542907714844,
      "rewards/margins": 1.1256234645843506,
      "rewards/rejected": -38.346168518066406,
      "step": 785
    },
    {
      "epoch": 0.4228131794614484,
      "grad_norm": 0.00024306372646533492,
      "learning_rate": 9.94920790594082e-07,
      "logits/chosen": 7.612584114074707,
      "logits/rejected": 7.527153015136719,
      "logps/chosen": -37.6441764831543,
      "logps/rejected": -40.033878326416016,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -37.6441764831543,
      "rewards/margins": 2.389700412750244,
      "rewards/rejected": -40.033878326416016,
      "step": 790
    },
    {
      "epoch": 0.42548921224284997,
      "grad_norm": 1.1333233573464902e-05,
      "learning_rate": 9.946969668401696e-07,
      "logits/chosen": 7.460398197174072,
      "logits/rejected": 6.999265193939209,
      "logps/chosen": -36.37274932861328,
      "logps/rejected": -37.866981506347656,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -36.37274932861328,
      "rewards/margins": 1.494228720664978,
      "rewards/rejected": -37.866981506347656,
      "step": 795
    },
    {
      "epoch": 0.4281652450242516,
      "grad_norm": 3.751260225097647e-10,
      "learning_rate": 9.944683435341155e-07,
      "logits/chosen": 7.543933868408203,
      "logits/rejected": 7.4145402908325195,
      "logps/chosen": -38.499019622802734,
      "logps/rejected": -39.2597541809082,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -38.499019622802734,
      "rewards/margins": 0.7607344388961792,
      "rewards/rejected": -39.2597541809082,
      "step": 800
    },
    {
      "epoch": 0.4281652450242516,
      "eval_logits/chosen": 6.36430549621582,
      "eval_logits/rejected": 6.230695724487305,
      "eval_logps/chosen": -37.04652404785156,
      "eval_logps/rejected": -38.52766418457031,
      "eval_loss": 1.0129234397027176e-05,
      "eval_rewards/accuracies": 0.5578634738922119,
      "eval_rewards/chosen": -37.04652404785156,
      "eval_rewards/margins": 1.4811420440673828,
      "eval_rewards/rejected": -38.52766418457031,
      "eval_runtime": 40.8465,
      "eval_samples_per_second": 32.928,
      "eval_steps_per_second": 8.25,
      "step": 800
    },
    {
      "epoch": 0.4308412778056531,
      "grad_norm": 0.008618508269670479,
      "learning_rate": 9.942349228940236e-07,
      "logits/chosen": 7.601966857910156,
      "logits/rejected": 7.381627082824707,
      "logps/chosen": -35.658599853515625,
      "logps/rejected": -39.303565979003906,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -35.658599853515625,
      "rewards/margins": 3.644965648651123,
      "rewards/rejected": -39.303565979003906,
      "step": 805
    },
    {
      "epoch": 0.43351731058705467,
      "grad_norm": 1.5419006373766908e-09,
      "learning_rate": 9.939967071845424e-07,
      "logits/chosen": 7.644564628601074,
      "logits/rejected": 7.591551303863525,
      "logps/chosen": -37.43498611450195,
      "logps/rejected": -38.69459533691406,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -37.43498611450195,
      "rewards/margins": 1.2596056461334229,
      "rewards/rejected": -38.69459533691406,
      "step": 810
    },
    {
      "epoch": 0.4361933433684563,
      "grad_norm": 6.957247575724642e-08,
      "learning_rate": 9.937536987168413e-07,
      "logits/chosen": 7.343323707580566,
      "logits/rejected": 7.149285316467285,
      "logps/chosen": -38.592342376708984,
      "logps/rejected": -41.225990295410156,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.592342376708984,
      "rewards/margins": 2.633650064468384,
      "rewards/rejected": -41.225990295410156,
      "step": 815
    },
    {
      "epoch": 0.4388693761498578,
      "grad_norm": 9.45959445445422e-06,
      "learning_rate": 9.935058998485896e-07,
      "logits/chosen": 7.62643575668335,
      "logits/rejected": 7.673081874847412,
      "logps/chosen": -39.537689208984375,
      "logps/rejected": -39.9161376953125,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -39.537689208984375,
      "rewards/margins": 0.3784486651420593,
      "rewards/rejected": -39.9161376953125,
      "step": 820
    },
    {
      "epoch": 0.44154540893125943,
      "grad_norm": 1.0012381106919758e-05,
      "learning_rate": 9.932533129839333e-07,
      "logits/chosen": 8.040262222290039,
      "logits/rejected": 7.694721221923828,
      "logps/chosen": -38.33998107910156,
      "logps/rejected": -39.52722930908203,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.33998107910156,
      "rewards/margins": 1.1872460842132568,
      "rewards/rejected": -39.52722930908203,
      "step": 825
    },
    {
      "epoch": 0.444221441712661,
      "grad_norm": 0.007977274137504639,
      "learning_rate": 9.929959405734711e-07,
      "logits/chosen": 7.41546106338501,
      "logits/rejected": 7.280850887298584,
      "logps/chosen": -36.20171356201172,
      "logps/rejected": -37.58140563964844,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -36.20171356201172,
      "rewards/margins": 1.3796923160552979,
      "rewards/rejected": -37.58140563964844,
      "step": 830
    },
    {
      "epoch": 0.44689747449406253,
      "grad_norm": 1.7754891368202017e-05,
      "learning_rate": 9.927337851142314e-07,
      "logits/chosen": 7.8377203941345215,
      "logits/rejected": 7.608137607574463,
      "logps/chosen": -38.19658660888672,
      "logps/rejected": -39.34210205078125,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -38.19658660888672,
      "rewards/margins": 1.1455156803131104,
      "rewards/rejected": -39.34210205078125,
      "step": 835
    },
    {
      "epoch": 0.44957350727546413,
      "grad_norm": 1.1258031598332168e-08,
      "learning_rate": 9.924668491496474e-07,
      "logits/chosen": 7.384390354156494,
      "logits/rejected": 6.984945774078369,
      "logps/chosen": -37.10817337036133,
      "logps/rejected": -39.376800537109375,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -37.10817337036133,
      "rewards/margins": 2.268624782562256,
      "rewards/rejected": -39.376800537109375,
      "step": 840
    },
    {
      "epoch": 0.4522495400568657,
      "grad_norm": 4.00140809333623e-05,
      "learning_rate": 9.92195135269533e-07,
      "logits/chosen": 7.49320125579834,
      "logits/rejected": 7.54547119140625,
      "logps/chosen": -37.49378204345703,
      "logps/rejected": -39.536685943603516,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -37.49378204345703,
      "rewards/margins": 2.0429091453552246,
      "rewards/rejected": -39.536685943603516,
      "step": 845
    },
    {
      "epoch": 0.4549255728382673,
      "grad_norm": 7.834505324589983e-07,
      "learning_rate": 9.919186461100574e-07,
      "logits/chosen": 7.840467929840088,
      "logits/rejected": 7.551217555999756,
      "logps/chosen": -37.86256790161133,
      "logps/rejected": -39.80552291870117,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -37.86256790161133,
      "rewards/margins": 1.9429543018341064,
      "rewards/rejected": -39.80552291870117,
      "step": 850
    },
    {
      "epoch": 0.45760160561966884,
      "grad_norm": 0.1767710140313415,
      "learning_rate": 9.9163738435372e-07,
      "logits/chosen": 7.156085014343262,
      "logits/rejected": 6.969692230224609,
      "logps/chosen": -35.7013053894043,
      "logps/rejected": -38.50730514526367,
      "loss": 0.0007,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -35.7013053894043,
      "rewards/margins": 2.805997848510742,
      "rewards/rejected": -38.50730514526367,
      "step": 855
    },
    {
      "epoch": 0.4602776384010704,
      "grad_norm": 2.3790709051049524e-05,
      "learning_rate": 9.913513527293234e-07,
      "logits/chosen": 7.813882350921631,
      "logits/rejected": 7.680117130279541,
      "logps/chosen": -36.60405731201172,
      "logps/rejected": -38.678749084472656,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -36.60405731201172,
      "rewards/margins": 2.074687957763672,
      "rewards/rejected": -38.678749084472656,
      "step": 860
    },
    {
      "epoch": 0.462953671182472,
      "grad_norm": 2.354378354385777e-07,
      "learning_rate": 9.910605540119474e-07,
      "logits/chosen": 7.874215126037598,
      "logits/rejected": 7.459391117095947,
      "logps/chosen": -39.80036163330078,
      "logps/rejected": -40.59810256958008,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -39.80036163330078,
      "rewards/margins": 0.7977336645126343,
      "rewards/rejected": -40.59810256958008,
      "step": 865
    },
    {
      "epoch": 0.46562970396387354,
      "grad_norm": 1.1259579120069062e-06,
      "learning_rate": 9.907649910229227e-07,
      "logits/chosen": 7.522549629211426,
      "logits/rejected": 7.291293144226074,
      "logps/chosen": -36.571571350097656,
      "logps/rejected": -39.611175537109375,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -36.571571350097656,
      "rewards/margins": 3.039602756500244,
      "rewards/rejected": -39.611175537109375,
      "step": 870
    },
    {
      "epoch": 0.46830573674527515,
      "grad_norm": 4.177973632904566e-06,
      "learning_rate": 9.90464666629803e-07,
      "logits/chosen": 7.758486747741699,
      "logits/rejected": 7.561933994293213,
      "logps/chosen": -37.98600769042969,
      "logps/rejected": -38.57429122924805,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -37.98600769042969,
      "rewards/margins": 0.5882858037948608,
      "rewards/rejected": -38.57429122924805,
      "step": 875
    },
    {
      "epoch": 0.4709817695266767,
      "grad_norm": 6.916912006007473e-05,
      "learning_rate": 9.901595837463363e-07,
      "logits/chosen": 7.254601955413818,
      "logits/rejected": 7.139842987060547,
      "logps/chosen": -35.60768127441406,
      "logps/rejected": -38.065185546875,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -35.60768127441406,
      "rewards/margins": 2.4575068950653076,
      "rewards/rejected": -38.065185546875,
      "step": 880
    },
    {
      "epoch": 0.47365780230807825,
      "grad_norm": 3.853764441528317e-07,
      "learning_rate": 9.898497453324384e-07,
      "logits/chosen": 7.788936614990234,
      "logits/rejected": 7.788862705230713,
      "logps/chosen": -38.87586212158203,
      "logps/rejected": -40.32879638671875,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -38.87586212158203,
      "rewards/margins": 1.4529345035552979,
      "rewards/rejected": -40.32879638671875,
      "step": 885
    },
    {
      "epoch": 0.47633383508947985,
      "grad_norm": 5.812918658075046e-06,
      "learning_rate": 9.895351543941628e-07,
      "logits/chosen": 7.8732171058654785,
      "logits/rejected": 7.65131139755249,
      "logps/chosen": -37.66066360473633,
      "logps/rejected": -39.59817123413086,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -37.66066360473633,
      "rewards/margins": 1.9375073909759521,
      "rewards/rejected": -39.59817123413086,
      "step": 890
    },
    {
      "epoch": 0.4790098678708814,
      "grad_norm": 1.0758085258701195e-08,
      "learning_rate": 9.892158139836724e-07,
      "logits/chosen": 7.62447452545166,
      "logits/rejected": 7.486349582672119,
      "logps/chosen": -38.606773376464844,
      "logps/rejected": -39.39204788208008,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -38.606773376464844,
      "rewards/margins": 0.785274863243103,
      "rewards/rejected": -39.39204788208008,
      "step": 895
    },
    {
      "epoch": 0.481685900652283,
      "grad_norm": 3.7952838189141804e-05,
      "learning_rate": 9.88891727199209e-07,
      "logits/chosen": 7.782861232757568,
      "logits/rejected": 7.703749656677246,
      "logps/chosen": -38.985172271728516,
      "logps/rejected": -41.012203216552734,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.985172271728516,
      "rewards/margins": 2.0270309448242188,
      "rewards/rejected": -41.012203216552734,
      "step": 900
    },
    {
      "epoch": 0.48436193343368455,
      "grad_norm": 0.0009951232639731184,
      "learning_rate": 9.885628971850641e-07,
      "logits/chosen": 7.884472846984863,
      "logits/rejected": 7.380224704742432,
      "logps/chosen": -38.044490814208984,
      "logps/rejected": -39.506675720214844,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -38.044490814208984,
      "rewards/margins": 1.4621862173080444,
      "rewards/rejected": -39.506675720214844,
      "step": 905
    },
    {
      "epoch": 0.48703796621508616,
      "grad_norm": 2.2052396655037205e-09,
      "learning_rate": 9.882293271315481e-07,
      "logits/chosen": 7.479225158691406,
      "logits/rejected": 7.292827606201172,
      "logps/chosen": -37.35040283203125,
      "logps/rejected": -38.60327911376953,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -37.35040283203125,
      "rewards/margins": 1.2528806924819946,
      "rewards/rejected": -38.60327911376953,
      "step": 910
    },
    {
      "epoch": 0.4897139989964877,
      "grad_norm": 0.0007963871998727564,
      "learning_rate": 9.878910202749589e-07,
      "logits/chosen": 7.811363220214844,
      "logits/rejected": 7.522627353668213,
      "logps/chosen": -36.3570442199707,
      "logps/rejected": -38.404640197753906,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -36.3570442199707,
      "rewards/margins": 2.0475997924804688,
      "rewards/rejected": -38.404640197753906,
      "step": 915
    },
    {
      "epoch": 0.49239003177788926,
      "grad_norm": 8.615165521933634e-05,
      "learning_rate": 9.875479798975512e-07,
      "logits/chosen": 7.5098395347595215,
      "logits/rejected": 6.997906684875488,
      "logps/chosen": -38.46096420288086,
      "logps/rejected": -40.547576904296875,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -38.46096420288086,
      "rewards/margins": 2.08661150932312,
      "rewards/rejected": -40.547576904296875,
      "step": 920
    },
    {
      "epoch": 0.49506606455929086,
      "grad_norm": 8.375688750634028e-07,
      "learning_rate": 9.87200209327504e-07,
      "logits/chosen": 7.657576084136963,
      "logits/rejected": 7.23116397857666,
      "logps/chosen": -38.20526885986328,
      "logps/rejected": -40.23979949951172,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.20526885986328,
      "rewards/margins": 2.0345311164855957,
      "rewards/rejected": -40.23979949951172,
      "step": 925
    },
    {
      "epoch": 0.4977420973406924,
      "grad_norm": 1.0649943186027018e-09,
      "learning_rate": 9.868477119388894e-07,
      "logits/chosen": 7.509026527404785,
      "logits/rejected": 7.3842878341674805,
      "logps/chosen": -37.2547607421875,
      "logps/rejected": -38.1758918762207,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -37.2547607421875,
      "rewards/margins": 0.9211313128471375,
      "rewards/rejected": -38.1758918762207,
      "step": 930
    },
    {
      "epoch": 0.500418130122094,
      "grad_norm": 4.323767311271579e-05,
      "learning_rate": 9.864904911516383e-07,
      "logits/chosen": 8.09333324432373,
      "logits/rejected": 7.901297092437744,
      "logps/chosen": -38.77185821533203,
      "logps/rejected": -40.321510314941406,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -38.77185821533203,
      "rewards/margins": 1.5496532917022705,
      "rewards/rejected": -40.321510314941406,
      "step": 935
    },
    {
      "epoch": 0.5030941629034956,
      "grad_norm": 0.011011078345763153,
      "learning_rate": 9.861285504315084e-07,
      "logits/chosen": 7.495965003967285,
      "logits/rejected": 7.350213050842285,
      "logps/chosen": -38.28999710083008,
      "logps/rejected": -39.809730529785156,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -38.28999710083008,
      "rewards/margins": 1.5197333097457886,
      "rewards/rejected": -39.809730529785156,
      "step": 940
    },
    {
      "epoch": 0.5057701956848971,
      "grad_norm": 2.5926991049482472e-05,
      "learning_rate": 9.857618932900502e-07,
      "logits/chosen": 8.081062316894531,
      "logits/rejected": 7.699568271636963,
      "logps/chosen": -38.677940368652344,
      "logps/rejected": -40.96052932739258,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -38.677940368652344,
      "rewards/margins": 2.2825827598571777,
      "rewards/rejected": -40.96052932739258,
      "step": 945
    },
    {
      "epoch": 0.5084462284662987,
      "grad_norm": 0.003033728417356917,
      "learning_rate": 9.853905232845727e-07,
      "logits/chosen": 7.655567169189453,
      "logits/rejected": 7.4965009689331055,
      "logps/chosen": -37.157310485839844,
      "logps/rejected": -39.4751091003418,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -37.157310485839844,
      "rewards/margins": 2.317800760269165,
      "rewards/rejected": -39.4751091003418,
      "step": 950
    },
    {
      "epoch": 0.5111222612477003,
      "grad_norm": 1.8303040967632359e-06,
      "learning_rate": 9.850144440181095e-07,
      "logits/chosen": 7.543534755706787,
      "logits/rejected": 7.293342590332031,
      "logps/chosen": -36.8066291809082,
      "logps/rejected": -39.609161376953125,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -36.8066291809082,
      "rewards/margins": 2.802532911300659,
      "rewards/rejected": -39.609161376953125,
      "step": 955
    },
    {
      "epoch": 0.5137982940291018,
      "grad_norm": 4.7089219400119935e-05,
      "learning_rate": 9.846336591393832e-07,
      "logits/chosen": 7.614388942718506,
      "logits/rejected": 7.3226470947265625,
      "logps/chosen": -37.35138702392578,
      "logps/rejected": -39.05803680419922,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -37.35138702392578,
      "rewards/margins": 1.7066415548324585,
      "rewards/rejected": -39.05803680419922,
      "step": 960
    },
    {
      "epoch": 0.5164743268105034,
      "grad_norm": 8.080497424132641e-08,
      "learning_rate": 9.842481723427704e-07,
      "logits/chosen": 7.483859062194824,
      "logits/rejected": 7.482878684997559,
      "logps/chosen": -36.816795349121094,
      "logps/rejected": -38.47077560424805,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -36.816795349121094,
      "rewards/margins": 1.6539726257324219,
      "rewards/rejected": -38.47077560424805,
      "step": 965
    },
    {
      "epoch": 0.519150359591905,
      "grad_norm": 0.00022005626828690273,
      "learning_rate": 9.838579873682658e-07,
      "logits/chosen": 7.405804634094238,
      "logits/rejected": 7.3831682205200195,
      "logps/chosen": -36.64482498168945,
      "logps/rejected": -37.14829635620117,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -36.64482498168945,
      "rewards/margins": 0.5034673810005188,
      "rewards/rejected": -37.14829635620117,
      "step": 970
    },
    {
      "epoch": 0.5218263923733065,
      "grad_norm": 3.524468606787021e-06,
      "learning_rate": 9.834631080014457e-07,
      "logits/chosen": 7.954876899719238,
      "logits/rejected": 7.514945030212402,
      "logps/chosen": -38.008880615234375,
      "logps/rejected": -38.99338912963867,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -38.008880615234375,
      "rewards/margins": 0.9845094680786133,
      "rewards/rejected": -38.99338912963867,
      "step": 975
    },
    {
      "epoch": 0.5245024251547081,
      "grad_norm": 2.817397889273523e-05,
      "learning_rate": 9.830635380734312e-07,
      "logits/chosen": 7.911994934082031,
      "logits/rejected": 7.618377685546875,
      "logps/chosen": -38.021732330322266,
      "logps/rejected": -40.397789001464844,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -38.021732330322266,
      "rewards/margins": 2.3760623931884766,
      "rewards/rejected": -40.397789001464844,
      "step": 980
    },
    {
      "epoch": 0.5271784579361097,
      "grad_norm": 0.0003054801353373189,
      "learning_rate": 9.826592814608517e-07,
      "logits/chosen": 7.941254615783691,
      "logits/rejected": 7.508225440979004,
      "logps/chosen": -39.09351348876953,
      "logps/rejected": -40.13032150268555,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.09351348876953,
      "rewards/margins": 1.0368082523345947,
      "rewards/rejected": -40.13032150268555,
      "step": 985
    },
    {
      "epoch": 0.5298544907175113,
      "grad_norm": 2.7692966238449743e-09,
      "learning_rate": 9.822503420858067e-07,
      "logits/chosen": 7.556215763092041,
      "logits/rejected": 7.690290927886963,
      "logps/chosen": -38.79026412963867,
      "logps/rejected": -38.26075744628906,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -38.79026412963867,
      "rewards/margins": -0.5295054316520691,
      "rewards/rejected": -38.26075744628906,
      "step": 990
    },
    {
      "epoch": 0.5325305234989128,
      "grad_norm": 1.3164274782238733e-05,
      "learning_rate": 9.818367239158277e-07,
      "logits/chosen": 7.353630065917969,
      "logits/rejected": 7.314842224121094,
      "logps/chosen": -38.19721221923828,
      "logps/rejected": -38.21575164794922,
      "loss": 0.0,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -38.19721221923828,
      "rewards/margins": 0.018535912036895752,
      "rewards/rejected": -38.21575164794922,
      "step": 995
    },
    {
      "epoch": 0.5352065562803144,
      "grad_norm": 8.117008705392227e-07,
      "learning_rate": 9.8141843096384e-07,
      "logits/chosen": 7.69936990737915,
      "logits/rejected": 7.194311618804932,
      "logps/chosen": -37.16504669189453,
      "logps/rejected": -39.00267791748047,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -37.16504669189453,
      "rewards/margins": 1.8376274108886719,
      "rewards/rejected": -39.00267791748047,
      "step": 1000
    },
    {
      "epoch": 0.537882589061716,
      "grad_norm": 0.009834807349041097,
      "learning_rate": 9.809954672881237e-07,
      "logits/chosen": 7.660292625427246,
      "logits/rejected": 7.279736518859863,
      "logps/chosen": -38.558616638183594,
      "logps/rejected": -39.17066192626953,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -38.558616638183594,
      "rewards/margins": 0.6120432019233704,
      "rewards/rejected": -39.17066192626953,
      "step": 1005
    },
    {
      "epoch": 0.5405586218431175,
      "grad_norm": 0.00012281036588404807,
      "learning_rate": 9.80567836992274e-07,
      "logits/chosen": 8.064266204833984,
      "logits/rejected": 7.85525369644165,
      "logps/chosen": -37.27289581298828,
      "logps/rejected": -39.949440002441406,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -37.27289581298828,
      "rewards/margins": 2.676543712615967,
      "rewards/rejected": -39.949440002441406,
      "step": 1010
    },
    {
      "epoch": 0.5432346546245191,
      "grad_norm": 1.660593368360222e-06,
      "learning_rate": 9.801355442251625e-07,
      "logits/chosen": 8.168622970581055,
      "logits/rejected": 7.666064262390137,
      "logps/chosen": -37.37153625488281,
      "logps/rejected": -38.426536560058594,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -37.37153625488281,
      "rewards/margins": 1.0550004243850708,
      "rewards/rejected": -38.426536560058594,
      "step": 1015
    },
    {
      "epoch": 0.5459106874059207,
      "grad_norm": 2.49350215950251e-06,
      "learning_rate": 9.796985931808949e-07,
      "logits/chosen": 7.922774314880371,
      "logits/rejected": 7.670487403869629,
      "logps/chosen": -38.050071716308594,
      "logps/rejected": -40.38166809082031,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -38.050071716308594,
      "rewards/margins": 2.331602096557617,
      "rewards/rejected": -40.38166809082031,
      "step": 1020
    },
    {
      "epoch": 0.5485867201873222,
      "grad_norm": 0.00012182959582150638,
      "learning_rate": 9.792569880987724e-07,
      "logits/chosen": 7.931715488433838,
      "logits/rejected": 7.495892524719238,
      "logps/chosen": -38.64578628540039,
      "logps/rejected": -39.6500244140625,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -38.64578628540039,
      "rewards/margins": 1.004235029220581,
      "rewards/rejected": -39.6500244140625,
      "step": 1025
    },
    {
      "epoch": 0.5512627529687238,
      "grad_norm": 0.361254533250141,
      "learning_rate": 9.788107332632493e-07,
      "logits/chosen": 7.760477542877197,
      "logits/rejected": 8.117753982543945,
      "logps/chosen": -38.537662506103516,
      "logps/rejected": -40.69624710083008,
      "loss": 0.0021,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -38.537662506103516,
      "rewards/margins": 2.1585824489593506,
      "rewards/rejected": -40.69624710083008,
      "step": 1030
    },
    {
      "epoch": 0.5539387857501255,
      "grad_norm": 0.00036584275977218176,
      "learning_rate": 9.783598330038924e-07,
      "logits/chosen": 8.067626953125,
      "logits/rejected": 7.869374752044678,
      "logps/chosen": -37.5782356262207,
      "logps/rejected": -39.36564254760742,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -37.5782356262207,
      "rewards/margins": 1.7874081134796143,
      "rewards/rejected": -39.36564254760742,
      "step": 1035
    },
    {
      "epoch": 0.5566148185315271,
      "grad_norm": 1.3050769761167753e-08,
      "learning_rate": 9.779042916953376e-07,
      "logits/chosen": 7.5154266357421875,
      "logits/rejected": 6.9996843338012695,
      "logps/chosen": -37.27549362182617,
      "logps/rejected": -38.62503433227539,
      "loss": 0.0001,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -37.27549362182617,
      "rewards/margins": 1.3495433330535889,
      "rewards/rejected": -38.62503433227539,
      "step": 1040
    },
    {
      "epoch": 0.5592908513129285,
      "grad_norm": 2.2266412298239093e-06,
      "learning_rate": 9.774441137572487e-07,
      "logits/chosen": 7.77196741104126,
      "logits/rejected": 7.535773277282715,
      "logps/chosen": -38.214019775390625,
      "logps/rejected": -40.248146057128906,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -38.214019775390625,
      "rewards/margins": 2.0341203212738037,
      "rewards/rejected": -40.248146057128906,
      "step": 1045
    },
    {
      "epoch": 0.5619668840943302,
      "grad_norm": 0.0004249368159004441,
      "learning_rate": 9.76979303654274e-07,
      "logits/chosen": 7.581748962402344,
      "logits/rejected": 7.272462368011475,
      "logps/chosen": -37.40267562866211,
      "logps/rejected": -38.54042053222656,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -37.40267562866211,
      "rewards/margins": 1.1377480030059814,
      "rewards/rejected": -38.54042053222656,
      "step": 1050
    },
    {
      "epoch": 0.5646429168757318,
      "grad_norm": 1.5354593928172725e-09,
      "learning_rate": 9.765098658960035e-07,
      "logits/chosen": 7.954503536224365,
      "logits/rejected": 8.149930000305176,
      "logps/chosen": -38.73603057861328,
      "logps/rejected": -40.9662971496582,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -38.73603057861328,
      "rewards/margins": 2.23026704788208,
      "rewards/rejected": -40.9662971496582,
      "step": 1055
    },
    {
      "epoch": 0.5673189496571333,
      "grad_norm": 0.16407495049696466,
      "learning_rate": 9.76035805036924e-07,
      "logits/chosen": 7.949182987213135,
      "logits/rejected": 7.690735816955566,
      "logps/chosen": -38.31816864013672,
      "logps/rejected": -40.112491607666016,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -38.31816864013672,
      "rewards/margins": 1.7943261861801147,
      "rewards/rejected": -40.112491607666016,
      "step": 1060
    },
    {
      "epoch": 0.5699949824385349,
      "grad_norm": 0.00029397863166676997,
      "learning_rate": 9.755571256763764e-07,
      "logits/chosen": 8.006070137023926,
      "logits/rejected": 7.897732734680176,
      "logps/chosen": -37.7571907043457,
      "logps/rejected": -40.086395263671875,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -37.7571907043457,
      "rewards/margins": 2.3292055130004883,
      "rewards/rejected": -40.086395263671875,
      "step": 1065
    },
    {
      "epoch": 0.5726710152199365,
      "grad_norm": 4.145959972074204e-07,
      "learning_rate": 9.750738324585097e-07,
      "logits/chosen": 7.74841833114624,
      "logits/rejected": 7.3475165367126465,
      "logps/chosen": -37.12546920776367,
      "logps/rejected": -39.638248443603516,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -37.12546920776367,
      "rewards/margins": 2.5127782821655273,
      "rewards/rejected": -39.638248443603516,
      "step": 1070
    },
    {
      "epoch": 0.5753470480013381,
      "grad_norm": 1.423495572407899e-06,
      "learning_rate": 9.74585930072237e-07,
      "logits/chosen": 8.111615180969238,
      "logits/rejected": 7.866713047027588,
      "logps/chosen": -39.933719635009766,
      "logps/rejected": -42.261451721191406,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -39.933719635009766,
      "rewards/margins": 2.3277344703674316,
      "rewards/rejected": -42.261451721191406,
      "step": 1075
    },
    {
      "epoch": 0.5780230807827396,
      "grad_norm": 0.009672790038437799,
      "learning_rate": 9.740934232511892e-07,
      "logits/chosen": 7.873568058013916,
      "logits/rejected": 7.647279262542725,
      "logps/chosen": -38.52336883544922,
      "logps/rejected": -39.57231521606445,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -38.52336883544922,
      "rewards/margins": 1.048934817314148,
      "rewards/rejected": -39.57231521606445,
      "step": 1080
    },
    {
      "epoch": 0.5806991135641412,
      "grad_norm": 2.2426491865256395e-07,
      "learning_rate": 9.735963167736698e-07,
      "logits/chosen": 8.265195846557617,
      "logits/rejected": 7.997408866882324,
      "logps/chosen": -39.147308349609375,
      "logps/rejected": -40.83783721923828,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -39.147308349609375,
      "rewards/margins": 1.6905357837677002,
      "rewards/rejected": -40.83783721923828,
      "step": 1085
    },
    {
      "epoch": 0.5833751463455428,
      "grad_norm": 2.984132368887616e-07,
      "learning_rate": 9.730946154626078e-07,
      "logits/chosen": 7.9652204513549805,
      "logits/rejected": 7.508782386779785,
      "logps/chosen": -38.911537170410156,
      "logps/rejected": -39.65976333618164,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -38.911537170410156,
      "rewards/margins": 0.7482300400733948,
      "rewards/rejected": -39.65976333618164,
      "step": 1090
    },
    {
      "epoch": 0.5860511791269443,
      "grad_norm": 2.009116813281513e-06,
      "learning_rate": 9.725883241855117e-07,
      "logits/chosen": 8.442087173461914,
      "logits/rejected": 8.048315048217773,
      "logps/chosen": -40.02463150024414,
      "logps/rejected": -41.45846939086914,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -40.02463150024414,
      "rewards/margins": 1.4338366985321045,
      "rewards/rejected": -41.45846939086914,
      "step": 1095
    },
    {
      "epoch": 0.5887272119083459,
      "grad_norm": 0.6139429115609526,
      "learning_rate": 9.720774478544218e-07,
      "logits/chosen": 7.953616142272949,
      "logits/rejected": 7.603008270263672,
      "logps/chosen": -39.673118591308594,
      "logps/rejected": -40.210662841796875,
      "loss": 0.0002,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -39.673118591308594,
      "rewards/margins": 0.5375412106513977,
      "rewards/rejected": -40.210662841796875,
      "step": 1100
    },
    {
      "epoch": 0.5914032446897475,
      "grad_norm": 5.826684640971938e-06,
      "learning_rate": 9.715619914258624e-07,
      "logits/chosen": 8.497035026550293,
      "logits/rejected": 8.352646827697754,
      "logps/chosen": -39.933937072753906,
      "logps/rejected": -40.52880859375,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -39.933937072753906,
      "rewards/margins": 0.5948749780654907,
      "rewards/rejected": -40.52880859375,
      "step": 1105
    },
    {
      "epoch": 0.594079277471149,
      "grad_norm": 7.44512141444027e-06,
      "learning_rate": 9.710419599007937e-07,
      "logits/chosen": 7.805276393890381,
      "logits/rejected": 7.424256801605225,
      "logps/chosen": -38.6500244140625,
      "logps/rejected": -39.21177673339844,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -38.6500244140625,
      "rewards/margins": 0.5617539882659912,
      "rewards/rejected": -39.21177673339844,
      "step": 1110
    },
    {
      "epoch": 0.5967553102525506,
      "grad_norm": 7.710652805952235e-06,
      "learning_rate": 9.705173583245643e-07,
      "logits/chosen": 8.37909984588623,
      "logits/rejected": 7.973823547363281,
      "logps/chosen": -40.06968307495117,
      "logps/rejected": -42.461734771728516,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -40.06968307495117,
      "rewards/margins": 2.392047166824341,
      "rewards/rejected": -42.461734771728516,
      "step": 1115
    },
    {
      "epoch": 0.5994313430339522,
      "grad_norm": 3.4132177908186556e-07,
      "learning_rate": 9.699881917868609e-07,
      "logits/chosen": 8.056367874145508,
      "logits/rejected": 7.825897216796875,
      "logps/chosen": -37.437705993652344,
      "logps/rejected": -39.452354431152344,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -37.437705993652344,
      "rewards/margins": 2.0146484375,
      "rewards/rejected": -39.452354431152344,
      "step": 1120
    },
    {
      "epoch": 0.6021073758153538,
      "grad_norm": 0.00020303080176862164,
      "learning_rate": 9.694544654216594e-07,
      "logits/chosen": 7.815061092376709,
      "logits/rejected": 7.363912105560303,
      "logps/chosen": -39.43842697143555,
      "logps/rejected": -40.97020721435547,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -39.43842697143555,
      "rewards/margins": 1.5317813158035278,
      "rewards/rejected": -40.97020721435547,
      "step": 1125
    },
    {
      "epoch": 0.6047834085967553,
      "grad_norm": 0.0003991712640830549,
      "learning_rate": 9.689161844071755e-07,
      "logits/chosen": 7.602792263031006,
      "logits/rejected": 7.502458095550537,
      "logps/chosen": -38.625186920166016,
      "logps/rejected": -40.96052932739258,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -38.625186920166016,
      "rewards/margins": 2.3353395462036133,
      "rewards/rejected": -40.96052932739258,
      "step": 1130
    },
    {
      "epoch": 0.6074594413781569,
      "grad_norm": 4.37903272490884e-05,
      "learning_rate": 9.683733539658138e-07,
      "logits/chosen": 7.982511043548584,
      "logits/rejected": 7.626869201660156,
      "logps/chosen": -37.98310089111328,
      "logps/rejected": -40.76325225830078,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -37.98310089111328,
      "rewards/margins": 2.7801480293273926,
      "rewards/rejected": -40.76325225830078,
      "step": 1135
    },
    {
      "epoch": 0.6101354741595585,
      "grad_norm": 1.2942584596614336e-05,
      "learning_rate": 9.678259793641178e-07,
      "logits/chosen": 8.08725643157959,
      "logits/rejected": 8.020898818969727,
      "logps/chosen": -39.81681442260742,
      "logps/rejected": -40.624794006347656,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -39.81681442260742,
      "rewards/margins": 0.8079781532287598,
      "rewards/rejected": -40.624794006347656,
      "step": 1140
    },
    {
      "epoch": 0.61281150694096,
      "grad_norm": 1.900377238233814e-09,
      "learning_rate": 9.672740659127183e-07,
      "logits/chosen": 8.23723030090332,
      "logits/rejected": 7.829659938812256,
      "logps/chosen": -39.133609771728516,
      "logps/rejected": -39.844322204589844,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -39.133609771728516,
      "rewards/margins": 0.7107132077217102,
      "rewards/rejected": -39.844322204589844,
      "step": 1145
    },
    {
      "epoch": 0.6154875397223616,
      "grad_norm": 1.9878388306590226e-10,
      "learning_rate": 9.667176189662818e-07,
      "logits/chosen": 8.174409866333008,
      "logits/rejected": 7.807578086853027,
      "logps/chosen": -39.63080596923828,
      "logps/rejected": -40.829383850097656,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -39.63080596923828,
      "rewards/margins": 1.1985732316970825,
      "rewards/rejected": -40.829383850097656,
      "step": 1150
    },
    {
      "epoch": 0.6181635725037632,
      "grad_norm": 2.50688062942319e-05,
      "learning_rate": 9.661566439234592e-07,
      "logits/chosen": 7.922749996185303,
      "logits/rejected": 7.881862640380859,
      "logps/chosen": -38.62043762207031,
      "logps/rejected": -40.07584762573242,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -38.62043762207031,
      "rewards/margins": 1.4554095268249512,
      "rewards/rejected": -40.07584762573242,
      "step": 1155
    },
    {
      "epoch": 0.6208396052851648,
      "grad_norm": 3.5467628971347876e-07,
      "learning_rate": 9.655911462268327e-07,
      "logits/chosen": 7.79284143447876,
      "logits/rejected": 7.658049583435059,
      "logps/chosen": -38.09325408935547,
      "logps/rejected": -40.65690231323242,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -38.09325408935547,
      "rewards/margins": 2.563648223876953,
      "rewards/rejected": -40.65690231323242,
      "step": 1160
    },
    {
      "epoch": 0.6235156380665663,
      "grad_norm": 1.793625003254257e-07,
      "learning_rate": 9.650211313628636e-07,
      "logits/chosen": 7.992042541503906,
      "logits/rejected": 7.954789161682129,
      "logps/chosen": -38.96379470825195,
      "logps/rejected": -40.213645935058594,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -38.96379470825195,
      "rewards/margins": 1.2498466968536377,
      "rewards/rejected": -40.213645935058594,
      "step": 1165
    },
    {
      "epoch": 0.6261916708479679,
      "grad_norm": 0.00020843590155329592,
      "learning_rate": 9.644466048618386e-07,
      "logits/chosen": 7.867042541503906,
      "logits/rejected": 7.492088317871094,
      "logps/chosen": -38.6095085144043,
      "logps/rejected": -39.39339065551758,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -38.6095085144043,
      "rewards/margins": 0.7838778495788574,
      "rewards/rejected": -39.39339065551758,
      "step": 1170
    },
    {
      "epoch": 0.6288677036293695,
      "grad_norm": 3.1773373791217627e-07,
      "learning_rate": 9.63867572297816e-07,
      "logits/chosen": 8.233621597290039,
      "logits/rejected": 7.841109275817871,
      "logps/chosen": -39.95270919799805,
      "logps/rejected": -41.25849533081055,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -39.95270919799805,
      "rewards/margins": 1.3057838678359985,
      "rewards/rejected": -41.25849533081055,
      "step": 1175
    },
    {
      "epoch": 0.631543736410771,
      "grad_norm": 2.6626753800657045e-09,
      "learning_rate": 9.632840392885727e-07,
      "logits/chosen": 8.159266471862793,
      "logits/rejected": 7.716169834136963,
      "logps/chosen": -38.42955780029297,
      "logps/rejected": -40.182743072509766,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.42955780029297,
      "rewards/margins": 1.7531821727752686,
      "rewards/rejected": -40.182743072509766,
      "step": 1180
    },
    {
      "epoch": 0.6342197691921726,
      "grad_norm": 0.0018605726208984858,
      "learning_rate": 9.626960114955483e-07,
      "logits/chosen": 8.13940143585205,
      "logits/rejected": 7.720973014831543,
      "logps/chosen": -37.29474639892578,
      "logps/rejected": -40.07600021362305,
      "loss": 0.0001,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -37.29474639892578,
      "rewards/margins": 2.7812552452087402,
      "rewards/rejected": -40.07600021362305,
      "step": 1185
    },
    {
      "epoch": 0.6368958019735742,
      "grad_norm": 3.918177352610572e-09,
      "learning_rate": 9.621034946237909e-07,
      "logits/chosen": 8.093598365783691,
      "logits/rejected": 7.705621242523193,
      "logps/chosen": -39.510284423828125,
      "logps/rejected": -41.15454864501953,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -39.510284423828125,
      "rewards/margins": 1.6442687511444092,
      "rewards/rejected": -41.15454864501953,
      "step": 1190
    },
    {
      "epoch": 0.6395718347549757,
      "grad_norm": 0.000251170090949831,
      "learning_rate": 9.615064944219021e-07,
      "logits/chosen": 8.17841911315918,
      "logits/rejected": 7.797991752624512,
      "logps/chosen": -40.805179595947266,
      "logps/rejected": -42.521812438964844,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -40.805179595947266,
      "rewards/margins": 1.7166383266448975,
      "rewards/rejected": -42.521812438964844,
      "step": 1195
    },
    {
      "epoch": 0.6422478675363773,
      "grad_norm": 1.0366767541922134e-07,
      "learning_rate": 9.609050166819803e-07,
      "logits/chosen": 8.052102088928223,
      "logits/rejected": 7.984274387359619,
      "logps/chosen": -39.85681915283203,
      "logps/rejected": -40.10163879394531,
      "loss": 0.001,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -39.85681915283203,
      "rewards/margins": 0.24481861293315887,
      "rewards/rejected": -40.10163879394531,
      "step": 1200
    },
    {
      "epoch": 0.6422478675363773,
      "eval_logits/chosen": 6.700042247772217,
      "eval_logits/rejected": 6.514923572540283,
      "eval_logps/chosen": -38.794212341308594,
      "eval_logps/rejected": -40.12665939331055,
      "eval_loss": 1.5090794249772443e-06,
      "eval_rewards/accuracies": 0.5667656064033508,
      "eval_rewards/chosen": -38.794212341308594,
      "eval_rewards/margins": 1.3324474096298218,
      "eval_rewards/rejected": -40.12665939331055,
      "eval_runtime": 40.2985,
      "eval_samples_per_second": 33.376,
      "eval_steps_per_second": 8.363,
      "step": 1200
    },
    {
      "epoch": 0.6449239003177789,
      "grad_norm": 4.571025649803181e-07,
      "learning_rate": 9.602990672395653e-07,
      "logits/chosen": 8.21595287322998,
      "logits/rejected": 7.726536750793457,
      "logps/chosen": -39.610347747802734,
      "logps/rejected": -41.17707443237305,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -39.610347747802734,
      "rewards/margins": 1.566732406616211,
      "rewards/rejected": -41.17707443237305,
      "step": 1205
    },
    {
      "epoch": 0.6475999330991805,
      "grad_norm": 1.3910108117174059e-09,
      "learning_rate": 9.59688651973581e-07,
      "logits/chosen": 8.231789588928223,
      "logits/rejected": 7.744787693023682,
      "logps/chosen": -39.04650115966797,
      "logps/rejected": -39.942771911621094,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -39.04650115966797,
      "rewards/margins": 0.8962669372558594,
      "rewards/rejected": -39.942771911621094,
      "step": 1210
    },
    {
      "epoch": 0.650275965880582,
      "grad_norm": 0.0007458526424624522,
      "learning_rate": 9.590737768062792e-07,
      "logits/chosen": 8.047931671142578,
      "logits/rejected": 7.630962371826172,
      "logps/chosen": -38.702293395996094,
      "logps/rejected": -40.36885452270508,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.702293395996094,
      "rewards/margins": 1.6665655374526978,
      "rewards/rejected": -40.36885452270508,
      "step": 1215
    },
    {
      "epoch": 0.6529519986619836,
      "grad_norm": 0.1610317626246477,
      "learning_rate": 9.584544477031816e-07,
      "logits/chosen": 7.843100070953369,
      "logits/rejected": 7.694161891937256,
      "logps/chosen": -39.58910369873047,
      "logps/rejected": -42.216697692871094,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -39.58910369873047,
      "rewards/margins": 2.6275923252105713,
      "rewards/rejected": -42.216697692871094,
      "step": 1220
    },
    {
      "epoch": 0.6556280314433852,
      "grad_norm": 3.554071556976578e-05,
      "learning_rate": 9.578306706730215e-07,
      "logits/chosen": 8.172025680541992,
      "logits/rejected": 7.747583866119385,
      "logps/chosen": -39.46088409423828,
      "logps/rejected": -41.235870361328125,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -39.46088409423828,
      "rewards/margins": 1.774987816810608,
      "rewards/rejected": -41.235870361328125,
      "step": 1225
    },
    {
      "epoch": 0.6583040642247867,
      "grad_norm": 1.6651937292717813e-07,
      "learning_rate": 9.572024517676865e-07,
      "logits/chosen": 8.174722671508789,
      "logits/rejected": 7.941262722015381,
      "logps/chosen": -39.199161529541016,
      "logps/rejected": -40.506752014160156,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -39.199161529541016,
      "rewards/margins": 1.307594895362854,
      "rewards/rejected": -40.506752014160156,
      "step": 1230
    },
    {
      "epoch": 0.6609800970061883,
      "grad_norm": 1.0343609113431195e-06,
      "learning_rate": 9.565697970821593e-07,
      "logits/chosen": 8.244317054748535,
      "logits/rejected": 7.794415473937988,
      "logps/chosen": -38.94886016845703,
      "logps/rejected": -40.545372009277344,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -38.94886016845703,
      "rewards/margins": 1.5965158939361572,
      "rewards/rejected": -40.545372009277344,
      "step": 1235
    },
    {
      "epoch": 0.6636561297875899,
      "grad_norm": 1.6902023490552281e-06,
      "learning_rate": 9.559327127544585e-07,
      "logits/chosen": 8.549825668334961,
      "logits/rejected": 8.43071174621582,
      "logps/chosen": -41.14146041870117,
      "logps/rejected": -42.68232345581055,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.14146041870117,
      "rewards/margins": 1.5408649444580078,
      "rewards/rejected": -42.68232345581055,
      "step": 1240
    },
    {
      "epoch": 0.6663321625689914,
      "grad_norm": 5.155827998256065e-10,
      "learning_rate": 9.552912049655789e-07,
      "logits/chosen": 8.480178833007812,
      "logits/rejected": 8.107583999633789,
      "logps/chosen": -40.715816497802734,
      "logps/rejected": -42.54690170288086,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -40.715816497802734,
      "rewards/margins": 1.8310868740081787,
      "rewards/rejected": -42.54690170288086,
      "step": 1245
    },
    {
      "epoch": 0.669008195350393,
      "grad_norm": 0.003137647159854406,
      "learning_rate": 9.546452799394315e-07,
      "logits/chosen": 8.568636894226074,
      "logits/rejected": 8.026914596557617,
      "logps/chosen": -40.63987350463867,
      "logps/rejected": -42.18573760986328,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -40.63987350463867,
      "rewards/margins": 1.5458616018295288,
      "rewards/rejected": -42.18573760986328,
      "step": 1250
    },
    {
      "epoch": 0.6716842281317946,
      "grad_norm": 4.967368470942391e-09,
      "learning_rate": 9.539949439427846e-07,
      "logits/chosen": 8.262842178344727,
      "logits/rejected": 8.109676361083984,
      "logps/chosen": -40.08157730102539,
      "logps/rejected": -41.22216796875,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -40.08157730102539,
      "rewards/margins": 1.140596628189087,
      "rewards/rejected": -41.22216796875,
      "step": 1255
    },
    {
      "epoch": 0.6743602609131962,
      "grad_norm": 1.654212667263991e-09,
      "learning_rate": 9.533402032852002e-07,
      "logits/chosen": 8.280901908874512,
      "logits/rejected": 7.905728340148926,
      "logps/chosen": -39.38500213623047,
      "logps/rejected": -41.9587287902832,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -39.38500213623047,
      "rewards/margins": 2.5737221240997314,
      "rewards/rejected": -41.9587287902832,
      "step": 1260
    },
    {
      "epoch": 0.6770362936945977,
      "grad_norm": 1.0892819823674152e-09,
      "learning_rate": 9.526810643189754e-07,
      "logits/chosen": 8.520188331604004,
      "logits/rejected": 7.725737571716309,
      "logps/chosen": -39.5770378112793,
      "logps/rejected": -41.63107681274414,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -39.5770378112793,
      "rewards/margins": 2.0540409088134766,
      "rewards/rejected": -41.63107681274414,
      "step": 1265
    },
    {
      "epoch": 0.6797123264759993,
      "grad_norm": 1.5056855392210347e-06,
      "learning_rate": 9.52017533439079e-07,
      "logits/chosen": 8.530022621154785,
      "logits/rejected": 8.410375595092773,
      "logps/chosen": -40.831825256347656,
      "logps/rejected": -41.7933235168457,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -40.831825256347656,
      "rewards/margins": 0.9614920616149902,
      "rewards/rejected": -41.7933235168457,
      "step": 1270
    },
    {
      "epoch": 0.6823883592574009,
      "grad_norm": 1.6898715210826791e-06,
      "learning_rate": 9.513496170830909e-07,
      "logits/chosen": 8.285804748535156,
      "logits/rejected": 8.108721733093262,
      "logps/chosen": -39.77947235107422,
      "logps/rejected": -40.9272346496582,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.77947235107422,
      "rewards/margins": 1.1477549076080322,
      "rewards/rejected": -40.9272346496582,
      "step": 1275
    },
    {
      "epoch": 0.6850643920388024,
      "grad_norm": 9.957210999909553e-07,
      "learning_rate": 9.506773217311382e-07,
      "logits/chosen": 7.924864768981934,
      "logits/rejected": 7.505575656890869,
      "logps/chosen": -39.848594665527344,
      "logps/rejected": -40.96566390991211,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -39.848594665527344,
      "rewards/margins": 1.1170703172683716,
      "rewards/rejected": -40.96566390991211,
      "step": 1280
    },
    {
      "epoch": 0.687740424820204,
      "grad_norm": 9.583693548457569e-08,
      "learning_rate": 9.500006539058334e-07,
      "logits/chosen": 8.455413818359375,
      "logits/rejected": 7.947080135345459,
      "logps/chosen": -40.40897750854492,
      "logps/rejected": -41.91022491455078,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -40.40897750854492,
      "rewards/margins": 1.5012447834014893,
      "rewards/rejected": -41.91022491455078,
      "step": 1285
    },
    {
      "epoch": 0.6904164576016056,
      "grad_norm": 2.677881002744384e-06,
      "learning_rate": 9.493196201722109e-07,
      "logits/chosen": 8.407316207885742,
      "logits/rejected": 8.076360702514648,
      "logps/chosen": -39.57369613647461,
      "logps/rejected": -41.303043365478516,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -39.57369613647461,
      "rewards/margins": 1.7293485403060913,
      "rewards/rejected": -41.303043365478516,
      "step": 1290
    },
    {
      "epoch": 0.6930924903830072,
      "grad_norm": 2.2516851606508294e-10,
      "learning_rate": 9.486342271376628e-07,
      "logits/chosen": 8.39858627319336,
      "logits/rejected": 8.682069778442383,
      "logps/chosen": -39.82408905029297,
      "logps/rejected": -41.20485305786133,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -39.82408905029297,
      "rewards/margins": 1.3807626962661743,
      "rewards/rejected": -41.20485305786133,
      "step": 1295
    },
    {
      "epoch": 0.6957685231644087,
      "grad_norm": 1.524749759736089e-07,
      "learning_rate": 9.479444814518755e-07,
      "logits/chosen": 8.44395923614502,
      "logits/rejected": 7.684889316558838,
      "logps/chosen": -39.279842376708984,
      "logps/rejected": -41.58313751220703,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -39.279842376708984,
      "rewards/margins": 2.303298234939575,
      "rewards/rejected": -41.58313751220703,
      "step": 1300
    },
    {
      "epoch": 0.6984445559458103,
      "grad_norm": 3.4551212633776652e-06,
      "learning_rate": 9.472503898067645e-07,
      "logits/chosen": 8.053625106811523,
      "logits/rejected": 8.036966323852539,
      "logps/chosen": -40.34284210205078,
      "logps/rejected": -40.94997024536133,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -40.34284210205078,
      "rewards/margins": 0.6071248650550842,
      "rewards/rejected": -40.94997024536133,
      "step": 1305
    },
    {
      "epoch": 0.701120588727212,
      "grad_norm": 2.1205545159748336e-09,
      "learning_rate": 9.465519589364099e-07,
      "logits/chosen": 7.924736976623535,
      "logits/rejected": 7.821497440338135,
      "logps/chosen": -38.628082275390625,
      "logps/rejected": -40.260337829589844,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -38.628082275390625,
      "rewards/margins": 1.63225519657135,
      "rewards/rejected": -40.260337829589844,
      "step": 1310
    },
    {
      "epoch": 0.7037966215086134,
      "grad_norm": 1.0614199759997344e-06,
      "learning_rate": 9.458491956169914e-07,
      "logits/chosen": 8.606477737426758,
      "logits/rejected": 8.16909408569336,
      "logps/chosen": -38.722190856933594,
      "logps/rejected": -41.011512756347656,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -38.722190856933594,
      "rewards/margins": 2.289320468902588,
      "rewards/rejected": -41.011512756347656,
      "step": 1315
    },
    {
      "epoch": 0.706472654290015,
      "grad_norm": 1.3301969072183396e-07,
      "learning_rate": 9.451421066667215e-07,
      "logits/chosen": 8.571816444396973,
      "logits/rejected": 8.233956336975098,
      "logps/chosen": -39.03949737548828,
      "logps/rejected": -41.33659744262695,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -39.03949737548828,
      "rewards/margins": 2.2970995903015137,
      "rewards/rejected": -41.33659744262695,
      "step": 1320
    },
    {
      "epoch": 0.7091486870714167,
      "grad_norm": 0.00019004317750762452,
      "learning_rate": 9.444306989457805e-07,
      "logits/chosen": 8.159810066223145,
      "logits/rejected": 8.031582832336426,
      "logps/chosen": -39.55509567260742,
      "logps/rejected": -41.02944564819336,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -39.55509567260742,
      "rewards/margins": 1.4743516445159912,
      "rewards/rejected": -41.02944564819336,
      "step": 1325
    },
    {
      "epoch": 0.7118247198528181,
      "grad_norm": 4.594547145170921e-09,
      "learning_rate": 9.437149793562489e-07,
      "logits/chosen": 8.045509338378906,
      "logits/rejected": 7.7881035804748535,
      "logps/chosen": -38.70356369018555,
      "logps/rejected": -39.1339225769043,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -38.70356369018555,
      "rewards/margins": 0.4303584098815918,
      "rewards/rejected": -39.1339225769043,
      "step": 1330
    },
    {
      "epoch": 0.7145007526342197,
      "grad_norm": 2.152601177913876e-05,
      "learning_rate": 9.429949548420417e-07,
      "logits/chosen": 8.452252388000488,
      "logits/rejected": 8.18472671508789,
      "logps/chosen": -39.76271057128906,
      "logps/rejected": -40.83562469482422,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -39.76271057128906,
      "rewards/margins": 1.0729191303253174,
      "rewards/rejected": -40.83562469482422,
      "step": 1335
    },
    {
      "epoch": 0.7171767854156214,
      "grad_norm": 2.4505040532485936,
      "learning_rate": 9.422706323888396e-07,
      "logits/chosen": 8.398438453674316,
      "logits/rejected": 8.44367790222168,
      "logps/chosen": -38.87419891357422,
      "logps/rejected": -40.024925231933594,
      "loss": 0.0003,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.87419891357422,
      "rewards/margins": 1.150724172592163,
      "rewards/rejected": -40.024925231933594,
      "step": 1340
    },
    {
      "epoch": 0.719852818197023,
      "grad_norm": 5.692852308569447e-05,
      "learning_rate": 9.415420190240225e-07,
      "logits/chosen": 8.081521987915039,
      "logits/rejected": 7.656586647033691,
      "logps/chosen": -36.452674865722656,
      "logps/rejected": -40.089290618896484,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -36.452674865722656,
      "rewards/margins": 3.6366171836853027,
      "rewards/rejected": -40.089290618896484,
      "step": 1345
    },
    {
      "epoch": 0.7225288509784245,
      "grad_norm": 0.6167234651032344,
      "learning_rate": 9.408091218166002e-07,
      "logits/chosen": 6.0876383781433105,
      "logits/rejected": 6.024787902832031,
      "logps/chosen": -30.829113006591797,
      "logps/rejected": -31.317214965820312,
      "loss": 0.0001,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -30.829113006591797,
      "rewards/margins": 0.4881015717983246,
      "rewards/rejected": -31.317214965820312,
      "step": 1350
    },
    {
      "epoch": 0.7252048837598261,
      "grad_norm": 0.0021453077032664494,
      "learning_rate": 9.400719478771449e-07,
      "logits/chosen": 6.615061283111572,
      "logits/rejected": 6.28089714050293,
      "logps/chosen": -33.01981735229492,
      "logps/rejected": -36.19890594482422,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -33.01981735229492,
      "rewards/margins": 3.1790847778320312,
      "rewards/rejected": -36.19890594482422,
      "step": 1355
    },
    {
      "epoch": 0.7278809165412277,
      "grad_norm": 7.938306104734793e-05,
      "learning_rate": 9.393305043577209e-07,
      "logits/chosen": 7.4625244140625,
      "logits/rejected": 7.271597385406494,
      "logps/chosen": -36.03832244873047,
      "logps/rejected": -36.957035064697266,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -36.03832244873047,
      "rewards/margins": 0.9187110662460327,
      "rewards/rejected": -36.957035064697266,
      "step": 1360
    },
    {
      "epoch": 0.7305569493226292,
      "grad_norm": 0.00015826475228190558,
      "learning_rate": 9.38584798451817e-07,
      "logits/chosen": 7.673105716705322,
      "logits/rejected": 7.417733192443848,
      "logps/chosen": -35.800697326660156,
      "logps/rejected": -38.100189208984375,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -35.800697326660156,
      "rewards/margins": 2.299489736557007,
      "rewards/rejected": -38.100189208984375,
      "step": 1365
    },
    {
      "epoch": 0.7332329821040308,
      "grad_norm": 8.869733324152551e-07,
      "learning_rate": 9.37834837394275e-07,
      "logits/chosen": 7.683876037597656,
      "logits/rejected": 7.434299468994141,
      "logps/chosen": -37.038326263427734,
      "logps/rejected": -39.22920608520508,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -37.038326263427734,
      "rewards/margins": 2.190875768661499,
      "rewards/rejected": -39.22920608520508,
      "step": 1370
    },
    {
      "epoch": 0.7359090148854324,
      "grad_norm": 1.388452303590589e-05,
      "learning_rate": 9.370806284612203e-07,
      "logits/chosen": 7.973485469818115,
      "logits/rejected": 7.760767936706543,
      "logps/chosen": -36.79266357421875,
      "logps/rejected": -39.08042526245117,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -36.79266357421875,
      "rewards/margins": 2.2877659797668457,
      "rewards/rejected": -39.08042526245117,
      "step": 1375
    },
    {
      "epoch": 0.738585047666834,
      "grad_norm": 1.8074447480362218e-06,
      "learning_rate": 9.363221789699912e-07,
      "logits/chosen": 8.076640129089355,
      "logits/rejected": 7.6811933517456055,
      "logps/chosen": -36.773460388183594,
      "logps/rejected": -38.93933868408203,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -36.773460388183594,
      "rewards/margins": 2.1658780574798584,
      "rewards/rejected": -38.93933868408203,
      "step": 1380
    },
    {
      "epoch": 0.7412610804482355,
      "grad_norm": 1.1053272986013867e-05,
      "learning_rate": 9.355594962790682e-07,
      "logits/chosen": 7.998993873596191,
      "logits/rejected": 7.6072235107421875,
      "logps/chosen": -37.936492919921875,
      "logps/rejected": -38.762691497802734,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -37.936492919921875,
      "rewards/margins": 0.8261971473693848,
      "rewards/rejected": -38.762691497802734,
      "step": 1385
    },
    {
      "epoch": 0.7439371132296371,
      "grad_norm": 4.095058226728124e-06,
      "learning_rate": 9.34792587788002e-07,
      "logits/chosen": 7.89907693862915,
      "logits/rejected": 7.527780055999756,
      "logps/chosen": -38.19247817993164,
      "logps/rejected": -39.80179214477539,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -38.19247817993164,
      "rewards/margins": 1.6093158721923828,
      "rewards/rejected": -39.80179214477539,
      "step": 1390
    },
    {
      "epoch": 0.7466131460110387,
      "grad_norm": 0.00011260936289540233,
      "learning_rate": 9.34021460937342e-07,
      "logits/chosen": 7.540017127990723,
      "logits/rejected": 7.481609344482422,
      "logps/chosen": -38.1087760925293,
      "logps/rejected": -38.50666046142578,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -38.1087760925293,
      "rewards/margins": 0.39788302779197693,
      "rewards/rejected": -38.50666046142578,
      "step": 1395
    },
    {
      "epoch": 0.7492891787924402,
      "grad_norm": 1.501187632566141e-08,
      "learning_rate": 9.332461232085646e-07,
      "logits/chosen": 7.9191179275512695,
      "logits/rejected": 7.414793491363525,
      "logps/chosen": -37.29340744018555,
      "logps/rejected": -38.46513748168945,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -37.29340744018555,
      "rewards/margins": 1.171724557876587,
      "rewards/rejected": -38.46513748168945,
      "step": 1400
    },
    {
      "epoch": 0.7519652115738418,
      "grad_norm": 1.046631814084412e-06,
      "learning_rate": 9.324665821239998e-07,
      "logits/chosen": 7.848758697509766,
      "logits/rejected": 7.459347724914551,
      "logps/chosen": -37.425498962402344,
      "logps/rejected": -38.769317626953125,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -37.425498962402344,
      "rewards/margins": 1.3438196182250977,
      "rewards/rejected": -38.769317626953125,
      "step": 1405
    },
    {
      "epoch": 0.7546412443552434,
      "grad_norm": 4.710895590921679e-08,
      "learning_rate": 9.316828452467583e-07,
      "logits/chosen": 7.838174343109131,
      "logits/rejected": 7.629962921142578,
      "logps/chosen": -38.402591705322266,
      "logps/rejected": -40.70613479614258,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -38.402591705322266,
      "rewards/margins": 2.303539752960205,
      "rewards/rejected": -40.70613479614258,
      "step": 1410
    },
    {
      "epoch": 0.7573172771366449,
      "grad_norm": 1.961317145990124e-06,
      "learning_rate": 9.30894920180659e-07,
      "logits/chosen": 7.770921230316162,
      "logits/rejected": 7.653811454772949,
      "logps/chosen": -37.28989791870117,
      "logps/rejected": -39.459571838378906,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -37.28989791870117,
      "rewards/margins": 2.1696786880493164,
      "rewards/rejected": -39.459571838378906,
      "step": 1415
    },
    {
      "epoch": 0.7599933099180465,
      "grad_norm": 3.0897937311184044e-05,
      "learning_rate": 9.301028145701543e-07,
      "logits/chosen": 7.6099958419799805,
      "logits/rejected": 7.298924922943115,
      "logps/chosen": -38.49353790283203,
      "logps/rejected": -40.264835357666016,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -38.49353790283203,
      "rewards/margins": 1.7713000774383545,
      "rewards/rejected": -40.264835357666016,
      "step": 1420
    },
    {
      "epoch": 0.7626693426994481,
      "grad_norm": 1.3305567730394303e-05,
      "learning_rate": 9.293065361002563e-07,
      "logits/chosen": 7.934554100036621,
      "logits/rejected": 7.427609443664551,
      "logps/chosen": -38.10251235961914,
      "logps/rejected": -38.60419845581055,
      "loss": 0.0021,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -38.10251235961914,
      "rewards/margins": 0.5016874074935913,
      "rewards/rejected": -38.60419845581055,
      "step": 1425
    },
    {
      "epoch": 0.7653453754808497,
      "grad_norm": 1.4062977409438572e-08,
      "learning_rate": 9.285060924964622e-07,
      "logits/chosen": 7.973842620849609,
      "logits/rejected": 7.8127593994140625,
      "logps/chosen": -38.819007873535156,
      "logps/rejected": -40.2440071105957,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -38.819007873535156,
      "rewards/margins": 1.4250024557113647,
      "rewards/rejected": -40.2440071105957,
      "step": 1430
    },
    {
      "epoch": 0.7680214082622512,
      "grad_norm": 9.975349442330303e-06,
      "learning_rate": 9.277014915246792e-07,
      "logits/chosen": 7.839162349700928,
      "logits/rejected": 7.7627105712890625,
      "logps/chosen": -38.88307189941406,
      "logps/rejected": -39.49738693237305,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -38.88307189941406,
      "rewards/margins": 0.6143161654472351,
      "rewards/rejected": -39.49738693237305,
      "step": 1435
    },
    {
      "epoch": 0.7706974410436528,
      "grad_norm": 8.637710104131475e-06,
      "learning_rate": 9.268927409911498e-07,
      "logits/chosen": 7.771187782287598,
      "logits/rejected": 7.478902339935303,
      "logps/chosen": -37.142513275146484,
      "logps/rejected": -38.183467864990234,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -37.142513275146484,
      "rewards/margins": 1.0409519672393799,
      "rewards/rejected": -38.183467864990234,
      "step": 1440
    },
    {
      "epoch": 0.7733734738250544,
      "grad_norm": 1.0452485738742639e-06,
      "learning_rate": 9.260798487423749e-07,
      "logits/chosen": 7.9516777992248535,
      "logits/rejected": 7.4535932540893555,
      "logps/chosen": -36.872398376464844,
      "logps/rejected": -39.068668365478516,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -36.872398376464844,
      "rewards/margins": 2.1962697505950928,
      "rewards/rejected": -39.068668365478516,
      "step": 1445
    },
    {
      "epoch": 0.7760495066064559,
      "grad_norm": 5.006829784080116e-06,
      "learning_rate": 9.252628226650389e-07,
      "logits/chosen": 7.831016540527344,
      "logits/rejected": 7.875929355621338,
      "logps/chosen": -38.368690490722656,
      "logps/rejected": -39.260658264160156,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -38.368690490722656,
      "rewards/margins": 0.8919647336006165,
      "rewards/rejected": -39.260658264160156,
      "step": 1450
    },
    {
      "epoch": 0.7787255393878575,
      "grad_norm": 5.184657435857802e-08,
      "learning_rate": 9.244416706859321e-07,
      "logits/chosen": 8.145885467529297,
      "logits/rejected": 7.772180080413818,
      "logps/chosen": -38.11443328857422,
      "logps/rejected": -40.17070770263672,
      "loss": 0.0004,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -38.11443328857422,
      "rewards/margins": 2.056274890899658,
      "rewards/rejected": -40.17070770263672,
      "step": 1455
    },
    {
      "epoch": 0.7814015721692591,
      "grad_norm": 1.0298486667660806e-07,
      "learning_rate": 9.23616400771875e-07,
      "logits/chosen": 8.182382583618164,
      "logits/rejected": 7.63820743560791,
      "logps/chosen": -38.30253601074219,
      "logps/rejected": -40.93608474731445,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -38.30253601074219,
      "rewards/margins": 2.633549213409424,
      "rewards/rejected": -40.93608474731445,
      "step": 1460
    },
    {
      "epoch": 0.7840776049506607,
      "grad_norm": 4.060329968153281e-09,
      "learning_rate": 9.227870209296395e-07,
      "logits/chosen": 7.824871063232422,
      "logits/rejected": 7.533638000488281,
      "logps/chosen": -37.28596496582031,
      "logps/rejected": -38.919158935546875,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -37.28596496582031,
      "rewards/margins": 1.6331911087036133,
      "rewards/rejected": -38.919158935546875,
      "step": 1465
    },
    {
      "epoch": 0.7867536377320622,
      "grad_norm": 6.307530129491754e-08,
      "learning_rate": 9.219535392058728e-07,
      "logits/chosen": 7.659951210021973,
      "logits/rejected": 7.760500907897949,
      "logps/chosen": -38.72264099121094,
      "logps/rejected": -39.74140167236328,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -38.72264099121094,
      "rewards/margins": 1.0187584161758423,
      "rewards/rejected": -39.74140167236328,
      "step": 1470
    },
    {
      "epoch": 0.7894296705134638,
      "grad_norm": 5.7826480907625454e-05,
      "learning_rate": 9.211159636870181e-07,
      "logits/chosen": 7.978939056396484,
      "logits/rejected": 7.410494327545166,
      "logps/chosen": -37.2846565246582,
      "logps/rejected": -39.65030288696289,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -37.2846565246582,
      "rewards/margins": 2.3656468391418457,
      "rewards/rejected": -39.65030288696289,
      "step": 1475
    },
    {
      "epoch": 0.7921057032948654,
      "grad_norm": 2.6875708712490975e-05,
      "learning_rate": 9.202743024992367e-07,
      "logits/chosen": 7.731909275054932,
      "logits/rejected": 7.61098575592041,
      "logps/chosen": -38.458248138427734,
      "logps/rejected": -40.160743713378906,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -38.458248138427734,
      "rewards/margins": 1.7024974822998047,
      "rewards/rejected": -40.160743713378906,
      "step": 1480
    },
    {
      "epoch": 0.7947817360762669,
      "grad_norm": 6.337849622040809e-06,
      "learning_rate": 9.194285638083293e-07,
      "logits/chosen": 7.647625923156738,
      "logits/rejected": 7.290806770324707,
      "logps/chosen": -36.965736389160156,
      "logps/rejected": -38.991966247558594,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -36.965736389160156,
      "rewards/margins": 2.0262298583984375,
      "rewards/rejected": -38.991966247558594,
      "step": 1485
    },
    {
      "epoch": 0.7974577688576685,
      "grad_norm": 7.976961720181852e-05,
      "learning_rate": 9.185787558196562e-07,
      "logits/chosen": 7.986492156982422,
      "logits/rejected": 7.838613986968994,
      "logps/chosen": -38.393680572509766,
      "logps/rejected": -40.16486358642578,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -38.393680572509766,
      "rewards/margins": 1.7711822986602783,
      "rewards/rejected": -40.16486358642578,
      "step": 1490
    },
    {
      "epoch": 0.8001338016390701,
      "grad_norm": 2.4953968429660495e-09,
      "learning_rate": 9.177248867780583e-07,
      "logits/chosen": 7.939672946929932,
      "logits/rejected": 7.871676445007324,
      "logps/chosen": -38.46574020385742,
      "logps/rejected": -39.33441925048828,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -38.46574020385742,
      "rewards/margins": 0.8686827421188354,
      "rewards/rejected": -39.33441925048828,
      "step": 1495
    },
    {
      "epoch": 0.8028098344204716,
      "grad_norm": 0.21504986252539593,
      "learning_rate": 9.168669649677769e-07,
      "logits/chosen": 7.964795112609863,
      "logits/rejected": 7.848135948181152,
      "logps/chosen": -38.21345520019531,
      "logps/rejected": -39.71274185180664,
      "loss": 0.0009,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -38.21345520019531,
      "rewards/margins": 1.4992845058441162,
      "rewards/rejected": -39.71274185180664,
      "step": 1500
    },
    {
      "epoch": 0.8054858672018732,
      "grad_norm": 7.833226036030094e-06,
      "learning_rate": 9.16004998712373e-07,
      "logits/chosen": 7.877539157867432,
      "logits/rejected": 7.718256950378418,
      "logps/chosen": -37.53843307495117,
      "logps/rejected": -39.277259826660156,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -37.53843307495117,
      "rewards/margins": 1.7388229370117188,
      "rewards/rejected": -39.277259826660156,
      "step": 1505
    },
    {
      "epoch": 0.8081618999832748,
      "grad_norm": 1.5945502243345816e-06,
      "learning_rate": 9.151389963746472e-07,
      "logits/chosen": 7.8832502365112305,
      "logits/rejected": 7.068793296813965,
      "logps/chosen": -37.3635368347168,
      "logps/rejected": -39.96464157104492,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -37.3635368347168,
      "rewards/margins": 2.6011085510253906,
      "rewards/rejected": -39.96464157104492,
      "step": 1510
    },
    {
      "epoch": 0.8108379327646764,
      "grad_norm": 1.4660878808362571e-06,
      "learning_rate": 9.142689663565577e-07,
      "logits/chosen": 7.9765191078186035,
      "logits/rejected": 7.9726762771606445,
      "logps/chosen": -38.6474609375,
      "logps/rejected": -39.47627639770508,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -38.6474609375,
      "rewards/margins": 0.8288129568099976,
      "rewards/rejected": -39.47627639770508,
      "step": 1515
    },
    {
      "epoch": 0.8135139655460779,
      "grad_norm": 5.661305889648504e-07,
      "learning_rate": 9.133949170991397e-07,
      "logits/chosen": 7.798492431640625,
      "logits/rejected": 7.728209018707275,
      "logps/chosen": -39.126319885253906,
      "logps/rejected": -40.037742614746094,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.126319885253906,
      "rewards/margins": 0.9114225506782532,
      "rewards/rejected": -40.037742614746094,
      "step": 1520
    },
    {
      "epoch": 0.8161899983274795,
      "grad_norm": 3.2416085141255877e-06,
      "learning_rate": 9.125168570824231e-07,
      "logits/chosen": 8.422554016113281,
      "logits/rejected": 7.839612007141113,
      "logps/chosen": -38.852500915527344,
      "logps/rejected": -39.63780212402344,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -38.852500915527344,
      "rewards/margins": 0.7852976322174072,
      "rewards/rejected": -39.63780212402344,
      "step": 1525
    },
    {
      "epoch": 0.8188660311088811,
      "grad_norm": 3.8940095013098934e-07,
      "learning_rate": 9.116347948253496e-07,
      "logits/chosen": 7.8123602867126465,
      "logits/rejected": 7.428183078765869,
      "logps/chosen": -36.50711441040039,
      "logps/rejected": -38.40597152709961,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -36.50711441040039,
      "rewards/margins": 1.8988609313964844,
      "rewards/rejected": -38.40597152709961,
      "step": 1530
    },
    {
      "epoch": 0.8215420638902826,
      "grad_norm": 7.654121182204357e-07,
      "learning_rate": 9.107487388856916e-07,
      "logits/chosen": 7.914336204528809,
      "logits/rejected": 7.348167419433594,
      "logps/chosen": -38.14690399169922,
      "logps/rejected": -39.80203628540039,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -38.14690399169922,
      "rewards/margins": 1.655135154724121,
      "rewards/rejected": -39.80203628540039,
      "step": 1535
    },
    {
      "epoch": 0.8242180966716842,
      "grad_norm": 2.167484314988425e-07,
      "learning_rate": 9.098586978599673e-07,
      "logits/chosen": 8.00414752960205,
      "logits/rejected": 7.612962245941162,
      "logps/chosen": -39.2923583984375,
      "logps/rejected": -40.04558181762695,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -39.2923583984375,
      "rewards/margins": 0.7532222867012024,
      "rewards/rejected": -40.04558181762695,
      "step": 1540
    },
    {
      "epoch": 0.8268941294530858,
      "grad_norm": 4.4633813092798436e-11,
      "learning_rate": 9.089646803833588e-07,
      "logits/chosen": 7.893210411071777,
      "logits/rejected": 7.460615634918213,
      "logps/chosen": -37.18971252441406,
      "logps/rejected": -39.01754379272461,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -37.18971252441406,
      "rewards/margins": 1.8278295993804932,
      "rewards/rejected": -39.01754379272461,
      "step": 1545
    },
    {
      "epoch": 0.8295701622344873,
      "grad_norm": 3.6015045712650233e-09,
      "learning_rate": 9.080666951296276e-07,
      "logits/chosen": 8.132731437683105,
      "logits/rejected": 7.5076704025268555,
      "logps/chosen": -37.65477752685547,
      "logps/rejected": -39.759315490722656,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -37.65477752685547,
      "rewards/margins": 2.1045339107513428,
      "rewards/rejected": -39.759315490722656,
      "step": 1550
    },
    {
      "epoch": 0.8322461950158889,
      "grad_norm": 1.7997245147723211e-07,
      "learning_rate": 9.071647508110305e-07,
      "logits/chosen": 8.195427894592285,
      "logits/rejected": 7.442620754241943,
      "logps/chosen": -36.724693298339844,
      "logps/rejected": -39.815242767333984,
      "loss": 0.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -36.724693298339844,
      "rewards/margins": 3.090546131134033,
      "rewards/rejected": -39.815242767333984,
      "step": 1555
    },
    {
      "epoch": 0.8349222277972905,
      "grad_norm": 6.536255965349954e-09,
      "learning_rate": 9.062588561782354e-07,
      "logits/chosen": 7.856629848480225,
      "logits/rejected": 7.8618903160095215,
      "logps/chosen": -40.22589111328125,
      "logps/rejected": -41.21640396118164,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -40.22589111328125,
      "rewards/margins": 0.9905115962028503,
      "rewards/rejected": -41.21640396118164,
      "step": 1560
    },
    {
      "epoch": 0.8375982605786921,
      "grad_norm": 8.558773747013863e-06,
      "learning_rate": 9.053490200202358e-07,
      "logits/chosen": 8.052000045776367,
      "logits/rejected": 8.14100170135498,
      "logps/chosen": -38.748878479003906,
      "logps/rejected": -39.23004913330078,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -38.748878479003906,
      "rewards/margins": 0.481168657541275,
      "rewards/rejected": -39.23004913330078,
      "step": 1565
    },
    {
      "epoch": 0.8402742933600936,
      "grad_norm": 1.384392143423722e-07,
      "learning_rate": 9.044352511642661e-07,
      "logits/chosen": 8.139077186584473,
      "logits/rejected": 7.994954586029053,
      "logps/chosen": -39.8129768371582,
      "logps/rejected": -40.227779388427734,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -39.8129768371582,
      "rewards/margins": 0.4148061275482178,
      "rewards/rejected": -40.227779388427734,
      "step": 1570
    },
    {
      "epoch": 0.8429503261414952,
      "grad_norm": 0.44767500323759146,
      "learning_rate": 9.03517558475716e-07,
      "logits/chosen": 7.806936740875244,
      "logits/rejected": 7.571896553039551,
      "logps/chosen": -37.864234924316406,
      "logps/rejected": -38.46239471435547,
      "loss": 0.0017,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -37.864234924316406,
      "rewards/margins": 0.5981568098068237,
      "rewards/rejected": -38.46239471435547,
      "step": 1575
    },
    {
      "epoch": 0.8456263589228968,
      "grad_norm": 9.082438208754553e-08,
      "learning_rate": 9.025959508580436e-07,
      "logits/chosen": 7.9717912673950195,
      "logits/rejected": 7.251823425292969,
      "logps/chosen": -37.87413024902344,
      "logps/rejected": -39.73711013793945,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -37.87413024902344,
      "rewards/margins": 1.8629745244979858,
      "rewards/rejected": -39.73711013793945,
      "step": 1580
    },
    {
      "epoch": 0.8483023917042983,
      "grad_norm": 0.00022946978330174063,
      "learning_rate": 9.016704372526905e-07,
      "logits/chosen": 8.054485321044922,
      "logits/rejected": 7.613073825836182,
      "logps/chosen": -37.721954345703125,
      "logps/rejected": -39.49103546142578,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -37.721954345703125,
      "rewards/margins": 1.7690833806991577,
      "rewards/rejected": -39.49103546142578,
      "step": 1585
    },
    {
      "epoch": 0.8509784244856999,
      "grad_norm": 9.462837610014242e-08,
      "learning_rate": 9.007410266389934e-07,
      "logits/chosen": 8.215543746948242,
      "logits/rejected": 8.285394668579102,
      "logps/chosen": -39.02608108520508,
      "logps/rejected": -40.97217559814453,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -39.02608108520508,
      "rewards/margins": 1.9460923671722412,
      "rewards/rejected": -40.97217559814453,
      "step": 1590
    },
    {
      "epoch": 0.8536544572671015,
      "grad_norm": 1.8100877377158613e-05,
      "learning_rate": 8.998077280340981e-07,
      "logits/chosen": 7.841731071472168,
      "logits/rejected": 7.768154144287109,
      "logps/chosen": -38.7288818359375,
      "logps/rejected": -39.626564025878906,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -38.7288818359375,
      "rewards/margins": 0.8976883888244629,
      "rewards/rejected": -39.626564025878906,
      "step": 1595
    },
    {
      "epoch": 0.8563304900485031,
      "grad_norm": 9.256304834389255e-06,
      "learning_rate": 8.988705504928722e-07,
      "logits/chosen": 8.257303237915039,
      "logits/rejected": 7.808728218078613,
      "logps/chosen": -38.027042388916016,
      "logps/rejected": -39.88721466064453,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.027042388916016,
      "rewards/margins": 1.8601722717285156,
      "rewards/rejected": -39.88721466064453,
      "step": 1600
    },
    {
      "epoch": 0.8563304900485031,
      "eval_logits/chosen": 6.747125148773193,
      "eval_logits/rejected": 6.570799350738525,
      "eval_logps/chosen": -38.59125900268555,
      "eval_logps/rejected": -40.01066207885742,
      "eval_loss": 8.234294455178315e-07,
      "eval_rewards/accuracies": 0.5667656064033508,
      "eval_rewards/chosen": -38.59125900268555,
      "eval_rewards/margins": 1.419403314590454,
      "eval_rewards/rejected": -40.01066207885742,
      "eval_runtime": 40.2701,
      "eval_samples_per_second": 33.399,
      "eval_steps_per_second": 8.368,
      "step": 1600
    },
    {
      "epoch": 0.8590065228299046,
      "grad_norm": 0.0008057370548501174,
      "learning_rate": 8.979295031078157e-07,
      "logits/chosen": 8.098664283752441,
      "logits/rejected": 7.638497829437256,
      "logps/chosen": -36.81837844848633,
      "logps/rejected": -39.67369842529297,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -36.81837844848633,
      "rewards/margins": 2.8553221225738525,
      "rewards/rejected": -39.67369842529297,
      "step": 1605
    },
    {
      "epoch": 0.8616825556113062,
      "grad_norm": 3.4653734778936245e-08,
      "learning_rate": 8.969845950089751e-07,
      "logits/chosen": 8.21097469329834,
      "logits/rejected": 7.8763885498046875,
      "logps/chosen": -38.55016326904297,
      "logps/rejected": -40.79650115966797,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -38.55016326904297,
      "rewards/margins": 2.2463347911834717,
      "rewards/rejected": -40.79650115966797,
      "step": 1610
    },
    {
      "epoch": 0.8643585883927078,
      "grad_norm": 3.510124901555926e-05,
      "learning_rate": 8.960358353638526e-07,
      "logits/chosen": 8.293069839477539,
      "logits/rejected": 7.962773323059082,
      "logps/chosen": -39.41699981689453,
      "logps/rejected": -40.80271530151367,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -39.41699981689453,
      "rewards/margins": 1.385716438293457,
      "rewards/rejected": -40.80271530151367,
      "step": 1615
    },
    {
      "epoch": 0.8670346211741093,
      "grad_norm": 1.3109905775399387e-06,
      "learning_rate": 8.950832333773184e-07,
      "logits/chosen": 8.098249435424805,
      "logits/rejected": 7.695958614349365,
      "logps/chosen": -39.975311279296875,
      "logps/rejected": -41.687217712402344,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -39.975311279296875,
      "rewards/margins": 1.7119166851043701,
      "rewards/rejected": -41.687217712402344,
      "step": 1620
    },
    {
      "epoch": 0.869710653955511,
      "grad_norm": 0.0008437320519301152,
      "learning_rate": 8.941267982915213e-07,
      "logits/chosen": 8.234050750732422,
      "logits/rejected": 7.85067892074585,
      "logps/chosen": -39.65726089477539,
      "logps/rejected": -38.728355407714844,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -39.65726089477539,
      "rewards/margins": -0.9288986325263977,
      "rewards/rejected": -38.728355407714844,
      "step": 1625
    },
    {
      "epoch": 0.8723866867369126,
      "grad_norm": 7.849080963198015e-06,
      "learning_rate": 8.931665393857983e-07,
      "logits/chosen": 8.340258598327637,
      "logits/rejected": 7.9926862716674805,
      "logps/chosen": -39.31509780883789,
      "logps/rejected": -41.00221633911133,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -39.31509780883789,
      "rewards/margins": 1.6871140003204346,
      "rewards/rejected": -41.00221633911133,
      "step": 1630
    },
    {
      "epoch": 0.875062719518314,
      "grad_norm": 3.6115655093926326e-11,
      "learning_rate": 8.922024659765861e-07,
      "logits/chosen": 8.636944770812988,
      "logits/rejected": 8.193704605102539,
      "logps/chosen": -39.755184173583984,
      "logps/rejected": -41.33856964111328,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -39.755184173583984,
      "rewards/margins": 1.5833840370178223,
      "rewards/rejected": -41.33856964111328,
      "step": 1635
    },
    {
      "epoch": 0.8777387522997157,
      "grad_norm": 1.1180178069155614e-09,
      "learning_rate": 8.912345874173288e-07,
      "logits/chosen": 8.38740062713623,
      "logits/rejected": 8.054380416870117,
      "logps/chosen": -39.5725212097168,
      "logps/rejected": -41.63652801513672,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -39.5725212097168,
      "rewards/margins": 2.06400990486145,
      "rewards/rejected": -41.63652801513672,
      "step": 1640
    },
    {
      "epoch": 0.8804147850811173,
      "grad_norm": 2.7386987524782393e-06,
      "learning_rate": 8.902629130983885e-07,
      "logits/chosen": 8.204886436462402,
      "logits/rejected": 8.130419731140137,
      "logps/chosen": -39.0472297668457,
      "logps/rejected": -39.94300079345703,
      "loss": 0.0006,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -39.0472297668457,
      "rewards/margins": 0.8957681655883789,
      "rewards/rejected": -39.94300079345703,
      "step": 1645
    },
    {
      "epoch": 0.8830908178625189,
      "grad_norm": 2.0202997218922968e-05,
      "learning_rate": 8.892874524469537e-07,
      "logits/chosen": 8.438112258911133,
      "logits/rejected": 8.220708847045898,
      "logps/chosen": -38.15906524658203,
      "logps/rejected": -38.746986389160156,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -38.15906524658203,
      "rewards/margins": 0.5879203081130981,
      "rewards/rejected": -38.746986389160156,
      "step": 1650
    },
    {
      "epoch": 0.8857668506439204,
      "grad_norm": 1.8190647719724933e-07,
      "learning_rate": 8.883082149269478e-07,
      "logits/chosen": 8.3302001953125,
      "logits/rejected": 8.150362968444824,
      "logps/chosen": -39.60652542114258,
      "logps/rejected": -40.55384063720703,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.60652542114258,
      "rewards/margins": 0.9473205804824829,
      "rewards/rejected": -40.55384063720703,
      "step": 1655
    },
    {
      "epoch": 0.888442883425322,
      "grad_norm": 6.371624494529953e-07,
      "learning_rate": 8.873252100389377e-07,
      "logits/chosen": 8.409173011779785,
      "logits/rejected": 8.225865364074707,
      "logps/chosen": -38.64631652832031,
      "logps/rejected": -39.588199615478516,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -38.64631652832031,
      "rewards/margins": 0.9418843984603882,
      "rewards/rejected": -39.588199615478516,
      "step": 1660
    },
    {
      "epoch": 0.8911189162067236,
      "grad_norm": 1.3970831175758207e-05,
      "learning_rate": 8.863384473200411e-07,
      "logits/chosen": 7.792959690093994,
      "logits/rejected": 7.656949043273926,
      "logps/chosen": -39.32639694213867,
      "logps/rejected": -40.312599182128906,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.32639694213867,
      "rewards/margins": 0.9861971735954285,
      "rewards/rejected": -40.312599182128906,
      "step": 1665
    },
    {
      "epoch": 0.8937949489881251,
      "grad_norm": 9.262325333340677e-07,
      "learning_rate": 8.853479363438342e-07,
      "logits/chosen": 8.590107917785645,
      "logits/rejected": 8.037742614746094,
      "logps/chosen": -40.97118377685547,
      "logps/rejected": -42.141761779785156,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -40.97118377685547,
      "rewards/margins": 1.170578956604004,
      "rewards/rejected": -42.141761779785156,
      "step": 1670
    },
    {
      "epoch": 0.8964709817695267,
      "grad_norm": 1.3293312236129075e-06,
      "learning_rate": 8.843536867202588e-07,
      "logits/chosen": 8.477182388305664,
      "logits/rejected": 8.070070266723633,
      "logps/chosen": -40.372779846191406,
      "logps/rejected": -41.76821517944336,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -40.372779846191406,
      "rewards/margins": 1.3954370021820068,
      "rewards/rejected": -41.76821517944336,
      "step": 1675
    },
    {
      "epoch": 0.8991470145509283,
      "grad_norm": 2.4413171514850148e-06,
      "learning_rate": 8.833557080955292e-07,
      "logits/chosen": 8.12739372253418,
      "logits/rejected": 7.710951805114746,
      "logps/chosen": -38.87654113769531,
      "logps/rejected": -39.08584976196289,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -38.87654113769531,
      "rewards/margins": 0.20930734276771545,
      "rewards/rejected": -39.08584976196289,
      "step": 1680
    },
    {
      "epoch": 0.9018230473323299,
      "grad_norm": 3.9459653876122e-06,
      "learning_rate": 8.823540101520381e-07,
      "logits/chosen": 8.33852767944336,
      "logits/rejected": 7.771746635437012,
      "logps/chosen": -39.13180160522461,
      "logps/rejected": -40.846595764160156,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -39.13180160522461,
      "rewards/margins": 1.7147928476333618,
      "rewards/rejected": -40.846595764160156,
      "step": 1685
    },
    {
      "epoch": 0.9044990801137314,
      "grad_norm": 0.0006194867369532477,
      "learning_rate": 8.813486026082637e-07,
      "logits/chosen": 8.267389297485352,
      "logits/rejected": 7.970789432525635,
      "logps/chosen": -39.289695739746094,
      "logps/rejected": -41.38640213012695,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -39.289695739746094,
      "rewards/margins": 2.096703290939331,
      "rewards/rejected": -41.38640213012695,
      "step": 1690
    },
    {
      "epoch": 0.907175112895133,
      "grad_norm": 6.549365291969407e-09,
      "learning_rate": 8.803394952186742e-07,
      "logits/chosen": 8.466710090637207,
      "logits/rejected": 8.094820022583008,
      "logps/chosen": -39.80694580078125,
      "logps/rejected": -42.62937927246094,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -39.80694580078125,
      "rewards/margins": 2.8224294185638428,
      "rewards/rejected": -42.62937927246094,
      "step": 1695
    },
    {
      "epoch": 0.9098511456765346,
      "grad_norm": 2.835247780281827e-10,
      "learning_rate": 8.793266977736342e-07,
      "logits/chosen": 7.980792999267578,
      "logits/rejected": 8.23906135559082,
      "logps/chosen": -40.428585052490234,
      "logps/rejected": -40.2591552734375,
      "loss": 0.0,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -40.428585052490234,
      "rewards/margins": -0.1694318801164627,
      "rewards/rejected": -40.2591552734375,
      "step": 1700
    },
    {
      "epoch": 0.9125271784579361,
      "grad_norm": 2.7940095700771216e-11,
      "learning_rate": 8.783102200993085e-07,
      "logits/chosen": 8.180706024169922,
      "logits/rejected": 8.084714889526367,
      "logps/chosen": -38.676658630371094,
      "logps/rejected": -40.654476165771484,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -38.676658630371094,
      "rewards/margins": 1.9778194427490234,
      "rewards/rejected": -40.654476165771484,
      "step": 1705
    },
    {
      "epoch": 0.9152032112393377,
      "grad_norm": 0.002906266536821842,
      "learning_rate": 8.772900720575683e-07,
      "logits/chosen": 8.309261322021484,
      "logits/rejected": 8.181894302368164,
      "logps/chosen": -38.73280334472656,
      "logps/rejected": -40.1246452331543,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -38.73280334472656,
      "rewards/margins": 1.3918428421020508,
      "rewards/rejected": -40.1246452331543,
      "step": 1710
    },
    {
      "epoch": 0.9178792440207393,
      "grad_norm": 1.1588609476828104e-07,
      "learning_rate": 8.762662635458944e-07,
      "logits/chosen": 8.500656127929688,
      "logits/rejected": 8.06970500946045,
      "logps/chosen": -40.282676696777344,
      "logps/rejected": -41.220420837402344,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -40.282676696777344,
      "rewards/margins": 0.937739372253418,
      "rewards/rejected": -41.220420837402344,
      "step": 1715
    },
    {
      "epoch": 0.9205552768021408,
      "grad_norm": 8.447053040242686e-07,
      "learning_rate": 8.752388044972811e-07,
      "logits/chosen": 7.965966701507568,
      "logits/rejected": 7.805613040924072,
      "logps/chosen": -38.75082778930664,
      "logps/rejected": -40.24550247192383,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.75082778930664,
      "rewards/margins": 1.4946743249893188,
      "rewards/rejected": -40.24550247192383,
      "step": 1720
    },
    {
      "epoch": 0.9232313095835424,
      "grad_norm": 8.514210680346087e-08,
      "learning_rate": 8.74207704880141e-07,
      "logits/chosen": 7.839042663574219,
      "logits/rejected": 7.6489458084106445,
      "logps/chosen": -38.32817840576172,
      "logps/rejected": -39.988704681396484,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -38.32817840576172,
      "rewards/margins": 1.6605284214019775,
      "rewards/rejected": -39.988704681396484,
      "step": 1725
    },
    {
      "epoch": 0.925907342364944,
      "grad_norm": 2.2854338711523137e-08,
      "learning_rate": 8.731729746982068e-07,
      "logits/chosen": 7.906767845153809,
      "logits/rejected": 7.990408897399902,
      "logps/chosen": -38.86058807373047,
      "logps/rejected": -40.13646697998047,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.86058807373047,
      "rewards/margins": 1.2758748531341553,
      "rewards/rejected": -40.13646697998047,
      "step": 1730
    },
    {
      "epoch": 0.9285833751463456,
      "grad_norm": 1.026963968553163e-09,
      "learning_rate": 8.721346239904355e-07,
      "logits/chosen": 8.595671653747559,
      "logits/rejected": 7.907167911529541,
      "logps/chosen": -38.16680908203125,
      "logps/rejected": -41.04495620727539,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -38.16680908203125,
      "rewards/margins": 2.8781521320343018,
      "rewards/rejected": -41.04495620727539,
      "step": 1735
    },
    {
      "epoch": 0.9312594079277471,
      "grad_norm": 2.0622232123449494e-07,
      "learning_rate": 8.710926628309101e-07,
      "logits/chosen": 8.425874710083008,
      "logits/rejected": 7.8293776512146,
      "logps/chosen": -38.79697799682617,
      "logps/rejected": -40.86872100830078,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -38.79697799682617,
      "rewards/margins": 2.0717406272888184,
      "rewards/rejected": -40.86872100830078,
      "step": 1740
    },
    {
      "epoch": 0.9339354407091487,
      "grad_norm": 7.205259694180774e-08,
      "learning_rate": 8.700471013287424e-07,
      "logits/chosen": 7.958510398864746,
      "logits/rejected": 7.977458953857422,
      "logps/chosen": -40.835960388183594,
      "logps/rejected": -40.94975280761719,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -40.835960388183594,
      "rewards/margins": 0.11378999799489975,
      "rewards/rejected": -40.94975280761719,
      "step": 1745
    },
    {
      "epoch": 0.9366114734905503,
      "grad_norm": 1.2549297995473493e-07,
      "learning_rate": 8.689979496279746e-07,
      "logits/chosen": 8.27040958404541,
      "logits/rejected": 8.281350135803223,
      "logps/chosen": -40.61737823486328,
      "logps/rejected": -41.963844299316406,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -40.61737823486328,
      "rewards/margins": 1.3464667797088623,
      "rewards/rejected": -41.963844299316406,
      "step": 1750
    },
    {
      "epoch": 0.9392875062719518,
      "grad_norm": 5.909337954162276e-10,
      "learning_rate": 8.679452179074811e-07,
      "logits/chosen": 8.432409286499023,
      "logits/rejected": 8.010265350341797,
      "logps/chosen": -39.39599609375,
      "logps/rejected": -40.23656463623047,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -39.39599609375,
      "rewards/margins": 0.8405696749687195,
      "rewards/rejected": -40.23656463623047,
      "step": 1755
    },
    {
      "epoch": 0.9419635390533534,
      "grad_norm": 6.976137089042503e-05,
      "learning_rate": 8.668889163808698e-07,
      "logits/chosen": 8.588744163513184,
      "logits/rejected": 8.143301010131836,
      "logps/chosen": -39.54351806640625,
      "logps/rejected": -40.878662109375,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -39.54351806640625,
      "rewards/margins": 1.3351447582244873,
      "rewards/rejected": -40.878662109375,
      "step": 1760
    },
    {
      "epoch": 0.944639571834755,
      "grad_norm": 1.538311254574968e-06,
      "learning_rate": 8.658290552963827e-07,
      "logits/chosen": 8.29702377319336,
      "logits/rejected": 8.092007637023926,
      "logps/chosen": -40.025142669677734,
      "logps/rejected": -40.387054443359375,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -40.025142669677734,
      "rewards/margins": 0.36191269755363464,
      "rewards/rejected": -40.387054443359375,
      "step": 1765
    },
    {
      "epoch": 0.9473156046161565,
      "grad_norm": 0.00038598147411679994,
      "learning_rate": 8.647656449367966e-07,
      "logits/chosen": 8.497171401977539,
      "logits/rejected": 8.19450855255127,
      "logps/chosen": -40.634422302246094,
      "logps/rejected": -41.51022720336914,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.634422302246094,
      "rewards/margins": 0.8758062124252319,
      "rewards/rejected": -41.51022720336914,
      "step": 1770
    },
    {
      "epoch": 0.9499916373975581,
      "grad_norm": 0.0012665874093443585,
      "learning_rate": 8.636986956193235e-07,
      "logits/chosen": 8.121536254882812,
      "logits/rejected": 7.710322380065918,
      "logps/chosen": -39.73516082763672,
      "logps/rejected": -40.877525329589844,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -39.73516082763672,
      "rewards/margins": 1.142362356185913,
      "rewards/rejected": -40.877525329589844,
      "step": 1775
    },
    {
      "epoch": 0.9526676701789597,
      "grad_norm": 2.821477919277297e-08,
      "learning_rate": 8.626282176955104e-07,
      "logits/chosen": 8.649648666381836,
      "logits/rejected": 8.281259536743164,
      "logps/chosen": -40.564125061035156,
      "logps/rejected": -42.40605545043945,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.564125061035156,
      "rewards/margins": 1.8419277667999268,
      "rewards/rejected": -42.40605545043945,
      "step": 1780
    },
    {
      "epoch": 0.9553437029603613,
      "grad_norm": 4.78053703715403e-11,
      "learning_rate": 8.615542215511389e-07,
      "logits/chosen": 8.362005233764648,
      "logits/rejected": 8.212501525878906,
      "logps/chosen": -40.52806091308594,
      "logps/rejected": -41.784908294677734,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -40.52806091308594,
      "rewards/margins": 1.2568451166152954,
      "rewards/rejected": -41.784908294677734,
      "step": 1785
    },
    {
      "epoch": 0.9580197357417628,
      "grad_norm": 4.030643790104539e-05,
      "learning_rate": 8.604767176061241e-07,
      "logits/chosen": 8.17466926574707,
      "logits/rejected": 7.714603424072266,
      "logps/chosen": -40.766319274902344,
      "logps/rejected": -41.360496520996094,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -40.766319274902344,
      "rewards/margins": 0.5941751599311829,
      "rewards/rejected": -41.360496520996094,
      "step": 1790
    },
    {
      "epoch": 0.9606957685231644,
      "grad_norm": 4.669020807058701e-10,
      "learning_rate": 8.593957163144141e-07,
      "logits/chosen": 8.524946212768555,
      "logits/rejected": 8.109848022460938,
      "logps/chosen": -40.80012893676758,
      "logps/rejected": -42.442359924316406,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -40.80012893676758,
      "rewards/margins": 1.6422319412231445,
      "rewards/rejected": -42.442359924316406,
      "step": 1795
    },
    {
      "epoch": 0.963371801304566,
      "grad_norm": 7.847506722154255e-06,
      "learning_rate": 8.58311228163888e-07,
      "logits/chosen": 8.733843803405762,
      "logits/rejected": 8.57422161102295,
      "logps/chosen": -40.84361267089844,
      "logps/rejected": -41.96295166015625,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -40.84361267089844,
      "rewards/margins": 1.119336724281311,
      "rewards/rejected": -41.96295166015625,
      "step": 1800
    },
    {
      "epoch": 0.9660478340859675,
      "grad_norm": 3.671573866661769e-05,
      "learning_rate": 8.57223263676255e-07,
      "logits/chosen": 9.078458786010742,
      "logits/rejected": 8.650040626525879,
      "logps/chosen": -40.602577209472656,
      "logps/rejected": -41.828426361083984,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -40.602577209472656,
      "rewards/margins": 1.225846290588379,
      "rewards/rejected": -41.828426361083984,
      "step": 1805
    },
    {
      "epoch": 0.9687238668673691,
      "grad_norm": 7.422031886836489e-07,
      "learning_rate": 8.561318334069511e-07,
      "logits/chosen": 8.545589447021484,
      "logits/rejected": 8.307339668273926,
      "logps/chosen": -41.22007751464844,
      "logps/rejected": -42.945213317871094,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -41.22007751464844,
      "rewards/margins": 1.7251379489898682,
      "rewards/rejected": -42.945213317871094,
      "step": 1810
    },
    {
      "epoch": 0.9713998996487707,
      "grad_norm": 1.1178607957254892e-10,
      "learning_rate": 8.550369479450375e-07,
      "logits/chosen": 8.53073501586914,
      "logits/rejected": 7.96996545791626,
      "logps/chosen": -39.37406539916992,
      "logps/rejected": -41.004798889160156,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -39.37406539916992,
      "rewards/margins": 1.6307334899902344,
      "rewards/rejected": -41.004798889160156,
      "step": 1815
    },
    {
      "epoch": 0.9740759324301723,
      "grad_norm": 5.870717612432264e-07,
      "learning_rate": 8.539386179130977e-07,
      "logits/chosen": 8.29672622680664,
      "logits/rejected": 8.00092887878418,
      "logps/chosen": -40.279754638671875,
      "logps/rejected": -40.98862075805664,
      "loss": 0.0001,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -40.279754638671875,
      "rewards/margins": 0.7088658213615417,
      "rewards/rejected": -40.98862075805664,
      "step": 1820
    },
    {
      "epoch": 0.9767519652115738,
      "grad_norm": 0.00023350833597350648,
      "learning_rate": 8.528368539671347e-07,
      "logits/chosen": 8.658112525939941,
      "logits/rejected": 7.9644269943237305,
      "logps/chosen": -42.06859588623047,
      "logps/rejected": -43.560813903808594,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.06859588623047,
      "rewards/margins": 1.4922200441360474,
      "rewards/rejected": -43.560813903808594,
      "step": 1825
    },
    {
      "epoch": 0.9794279979929754,
      "grad_norm": 6.5193348088399285e-06,
      "learning_rate": 8.51731666796467e-07,
      "logits/chosen": 7.822071075439453,
      "logits/rejected": 7.752615451812744,
      "logps/chosen": -39.534393310546875,
      "logps/rejected": -40.040592193603516,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.534393310546875,
      "rewards/margins": 0.5061955451965332,
      "rewards/rejected": -40.040592193603516,
      "step": 1830
    },
    {
      "epoch": 0.982104030774377,
      "grad_norm": 8.449476243670447e-08,
      "learning_rate": 8.506230671236254e-07,
      "logits/chosen": 8.228866577148438,
      "logits/rejected": 7.923471927642822,
      "logps/chosen": -40.069217681884766,
      "logps/rejected": -41.77763366699219,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.069217681884766,
      "rewards/margins": 1.7084156274795532,
      "rewards/rejected": -41.77763366699219,
      "step": 1835
    },
    {
      "epoch": 0.9847800635557785,
      "grad_norm": 9.304173878303832e-08,
      "learning_rate": 8.495110657042488e-07,
      "logits/chosen": 8.36742115020752,
      "logits/rejected": 7.849230766296387,
      "logps/chosen": -38.560062408447266,
      "logps/rejected": -40.37285614013672,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -38.560062408447266,
      "rewards/margins": 1.8127937316894531,
      "rewards/rejected": -40.37285614013672,
      "step": 1840
    },
    {
      "epoch": 0.9874560963371801,
      "grad_norm": 4.910410273649498e-10,
      "learning_rate": 8.483956733269799e-07,
      "logits/chosen": 8.148832321166992,
      "logits/rejected": 8.080046653747559,
      "logps/chosen": -40.808345794677734,
      "logps/rejected": -41.62120819091797,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -40.808345794677734,
      "rewards/margins": 0.8128618001937866,
      "rewards/rejected": -41.62120819091797,
      "step": 1845
    },
    {
      "epoch": 0.9901321291185817,
      "grad_norm": 4.5991306567031894e-09,
      "learning_rate": 8.472769008133602e-07,
      "logits/chosen": 8.324226379394531,
      "logits/rejected": 8.336076736450195,
      "logps/chosen": -38.10661315917969,
      "logps/rejected": -40.609596252441406,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -38.10661315917969,
      "rewards/margins": 2.502983570098877,
      "rewards/rejected": -40.609596252441406,
      "step": 1850
    },
    {
      "epoch": 0.9928081618999832,
      "grad_norm": 2.2668849239772056e-10,
      "learning_rate": 8.461547590177259e-07,
      "logits/chosen": 8.491576194763184,
      "logits/rejected": 8.102415084838867,
      "logps/chosen": -40.452545166015625,
      "logps/rejected": -42.19157409667969,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -40.452545166015625,
      "rewards/margins": 1.7390296459197998,
      "rewards/rejected": -42.19157409667969,
      "step": 1855
    },
    {
      "epoch": 0.9954841946813848,
      "grad_norm": 1.5169345964498458e-10,
      "learning_rate": 8.450292588271014e-07,
      "logits/chosen": 8.413893699645996,
      "logits/rejected": 8.100334167480469,
      "logps/chosen": -40.15303421020508,
      "logps/rejected": -41.212860107421875,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -40.15303421020508,
      "rewards/margins": 1.0598270893096924,
      "rewards/rejected": -41.212860107421875,
      "step": 1860
    },
    {
      "epoch": 0.9981602274627864,
      "grad_norm": 0.12548497991369065,
      "learning_rate": 8.439004111610945e-07,
      "logits/chosen": 8.626049041748047,
      "logits/rejected": 8.352197647094727,
      "logps/chosen": -40.972110748291016,
      "logps/rejected": -41.02737045288086,
      "loss": 0.0004,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -40.972110748291016,
      "rewards/margins": 0.0552624836564064,
      "rewards/rejected": -41.02737045288086,
      "step": 1865
    },
    {
      "epoch": 1.000836260244188,
      "grad_norm": 1.9962778138718993e-08,
      "learning_rate": 8.427682269717901e-07,
      "logits/chosen": 8.327581405639648,
      "logits/rejected": 7.998424530029297,
      "logps/chosen": -39.47612380981445,
      "logps/rejected": -40.70171356201172,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -39.47612380981445,
      "rewards/margins": 1.2255887985229492,
      "rewards/rejected": -40.70171356201172,
      "step": 1870
    },
    {
      "epoch": 1.0035122930255895,
      "grad_norm": 1.5439795048349868e-05,
      "learning_rate": 8.416327172436446e-07,
      "logits/chosen": 8.931873321533203,
      "logits/rejected": 8.444761276245117,
      "logps/chosen": -40.64853286743164,
      "logps/rejected": -42.009578704833984,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -40.64853286743164,
      "rewards/margins": 1.3610453605651855,
      "rewards/rejected": -42.009578704833984,
      "step": 1875
    },
    {
      "epoch": 1.0061883258069912,
      "grad_norm": 3.869508475276221e-10,
      "learning_rate": 8.404938929933778e-07,
      "logits/chosen": 8.344380378723145,
      "logits/rejected": 8.09056282043457,
      "logps/chosen": -41.148170471191406,
      "logps/rejected": -42.11793899536133,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -41.148170471191406,
      "rewards/margins": 0.9697696566581726,
      "rewards/rejected": -42.11793899536133,
      "step": 1880
    },
    {
      "epoch": 1.0088643585883927,
      "grad_norm": 1.4687148793435951e-07,
      "learning_rate": 8.39351765269868e-07,
      "logits/chosen": 8.577982902526855,
      "logits/rejected": 8.572615623474121,
      "logps/chosen": -41.54861831665039,
      "logps/rejected": -42.00185012817383,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.54861831665039,
      "rewards/margins": 0.4532257914543152,
      "rewards/rejected": -42.00185012817383,
      "step": 1885
    },
    {
      "epoch": 1.0115403913697942,
      "grad_norm": 5.804577554401265e-09,
      "learning_rate": 8.382063451540431e-07,
      "logits/chosen": 8.335451126098633,
      "logits/rejected": 7.899486541748047,
      "logps/chosen": -39.401573181152344,
      "logps/rejected": -40.882843017578125,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -39.401573181152344,
      "rewards/margins": 1.4812655448913574,
      "rewards/rejected": -40.882843017578125,
      "step": 1890
    },
    {
      "epoch": 1.014216424151196,
      "grad_norm": 0.00020324075539934118,
      "learning_rate": 8.370576437587742e-07,
      "logits/chosen": 8.167135238647461,
      "logits/rejected": 8.198922157287598,
      "logps/chosen": -39.173675537109375,
      "logps/rejected": -40.43407440185547,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -39.173675537109375,
      "rewards/margins": 1.2603999376296997,
      "rewards/rejected": -40.43407440185547,
      "step": 1895
    },
    {
      "epoch": 1.0168924569325974,
      "grad_norm": 3.701296079733634e-05,
      "learning_rate": 8.359056722287674e-07,
      "logits/chosen": 9.065496444702148,
      "logits/rejected": 8.296707153320312,
      "logps/chosen": -39.73955535888672,
      "logps/rejected": -41.647064208984375,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -39.73955535888672,
      "rewards/margins": 1.9075090885162354,
      "rewards/rejected": -41.647064208984375,
      "step": 1900
    },
    {
      "epoch": 1.019568489713999,
      "grad_norm": 1.5736020436401978e-06,
      "learning_rate": 8.347504417404553e-07,
      "logits/chosen": 8.390199661254883,
      "logits/rejected": 8.037742614746094,
      "logps/chosen": -39.66211700439453,
      "logps/rejected": -41.27189636230469,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -39.66211700439453,
      "rewards/margins": 1.6097815036773682,
      "rewards/rejected": -41.27189636230469,
      "step": 1905
    },
    {
      "epoch": 1.0222445224954007,
      "grad_norm": 6.755093236349187e-07,
      "learning_rate": 8.335919635018893e-07,
      "logits/chosen": 8.552318572998047,
      "logits/rejected": 8.313759803771973,
      "logps/chosen": -40.83061981201172,
      "logps/rejected": -41.274044036865234,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -40.83061981201172,
      "rewards/margins": 0.4434239864349365,
      "rewards/rejected": -41.274044036865234,
      "step": 1910
    },
    {
      "epoch": 1.0249205552768021,
      "grad_norm": 2.731198978173995e-07,
      "learning_rate": 8.324302487526303e-07,
      "logits/chosen": 8.598410606384277,
      "logits/rejected": 8.269886016845703,
      "logps/chosen": -40.91318893432617,
      "logps/rejected": -41.74647521972656,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -40.91318893432617,
      "rewards/margins": 0.8332853317260742,
      "rewards/rejected": -41.74647521972656,
      "step": 1915
    },
    {
      "epoch": 1.0275965880582036,
      "grad_norm": 8.397767482558755e-08,
      "learning_rate": 8.312653087636398e-07,
      "logits/chosen": 8.798726081848145,
      "logits/rejected": 8.53149700164795,
      "logps/chosen": -41.24126434326172,
      "logps/rejected": -42.506263732910156,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -41.24126434326172,
      "rewards/margins": 1.264999508857727,
      "rewards/rejected": -42.506263732910156,
      "step": 1920
    },
    {
      "epoch": 1.0302726208396054,
      "grad_norm": 1.573052666134527e-08,
      "learning_rate": 8.300971548371711e-07,
      "logits/chosen": 8.764241218566895,
      "logits/rejected": 8.133979797363281,
      "logps/chosen": -39.53509521484375,
      "logps/rejected": -41.868133544921875,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -39.53509521484375,
      "rewards/margins": 2.333033561706543,
      "rewards/rejected": -41.868133544921875,
      "step": 1925
    },
    {
      "epoch": 1.0329486536210069,
      "grad_norm": 4.044770848959594e-09,
      "learning_rate": 8.289257983066582e-07,
      "logits/chosen": 8.660727500915527,
      "logits/rejected": 8.264470100402832,
      "logps/chosen": -40.130043029785156,
      "logps/rejected": -42.04378128051758,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.130043029785156,
      "rewards/margins": 1.9137340784072876,
      "rewards/rejected": -42.04378128051758,
      "step": 1930
    },
    {
      "epoch": 1.0356246864024083,
      "grad_norm": 6.319664740266095e-07,
      "learning_rate": 8.277512505366077e-07,
      "logits/chosen": 8.515426635742188,
      "logits/rejected": 8.033183097839355,
      "logps/chosen": -39.58656311035156,
      "logps/rejected": -42.13504409790039,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -39.58656311035156,
      "rewards/margins": 2.548483371734619,
      "rewards/rejected": -42.13504409790039,
      "step": 1935
    },
    {
      "epoch": 1.03830071918381,
      "grad_norm": 4.165473375068464e-09,
      "learning_rate": 8.265735229224868e-07,
      "logits/chosen": 8.569108963012695,
      "logits/rejected": 8.4539794921875,
      "logps/chosen": -40.05710220336914,
      "logps/rejected": -42.40205383300781,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -40.05710220336914,
      "rewards/margins": 2.3449490070343018,
      "rewards/rejected": -42.40205383300781,
      "step": 1940
    },
    {
      "epoch": 1.0409767519652116,
      "grad_norm": 1.0050864873043155e-09,
      "learning_rate": 8.253926268906144e-07,
      "logits/chosen": 8.463804244995117,
      "logits/rejected": 8.174898147583008,
      "logps/chosen": -39.908111572265625,
      "logps/rejected": -42.075035095214844,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -39.908111572265625,
      "rewards/margins": 2.1669223308563232,
      "rewards/rejected": -42.075035095214844,
      "step": 1945
    },
    {
      "epoch": 1.043652784746613,
      "grad_norm": 0.2002540350699298,
      "learning_rate": 8.242085738980487e-07,
      "logits/chosen": 8.658209800720215,
      "logits/rejected": 7.952520847320557,
      "logps/chosen": -40.68439865112305,
      "logps/rejected": -42.43914794921875,
      "loss": 0.0008,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.68439865112305,
      "rewards/margins": 1.7547495365142822,
      "rewards/rejected": -42.43914794921875,
      "step": 1950
    },
    {
      "epoch": 1.0463288175280148,
      "grad_norm": 1.309803159323349e-08,
      "learning_rate": 8.230213754324772e-07,
      "logits/chosen": 8.807183265686035,
      "logits/rejected": 8.531861305236816,
      "logps/chosen": -40.243247985839844,
      "logps/rejected": -41.38704299926758,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -40.243247985839844,
      "rewards/margins": 1.143794298171997,
      "rewards/rejected": -41.38704299926758,
      "step": 1955
    },
    {
      "epoch": 1.0490048503094163,
      "grad_norm": 3.0880972689900786e-08,
      "learning_rate": 8.218310430121045e-07,
      "logits/chosen": 8.500699996948242,
      "logits/rejected": 8.387083053588867,
      "logps/chosen": -40.80499267578125,
      "logps/rejected": -41.40040969848633,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -40.80499267578125,
      "rewards/margins": 0.5954194664955139,
      "rewards/rejected": -41.40040969848633,
      "step": 1960
    },
    {
      "epoch": 1.051680883090818,
      "grad_norm": 2.1538840832711535e-09,
      "learning_rate": 8.20637588185541e-07,
      "logits/chosen": 8.188100814819336,
      "logits/rejected": 8.009719848632812,
      "logps/chosen": -39.265132904052734,
      "logps/rejected": -40.734710693359375,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -39.265132904052734,
      "rewards/margins": 1.4695723056793213,
      "rewards/rejected": -40.734710693359375,
      "step": 1965
    },
    {
      "epoch": 1.0543569158722195,
      "grad_norm": 2.799850130923686e-06,
      "learning_rate": 8.194410225316906e-07,
      "logits/chosen": 8.669782638549805,
      "logits/rejected": 8.101269721984863,
      "logps/chosen": -40.27177810668945,
      "logps/rejected": -41.751136779785156,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -40.27177810668945,
      "rewards/margins": 1.4793590307235718,
      "rewards/rejected": -41.751136779785156,
      "step": 1970
    },
    {
      "epoch": 1.057032948653621,
      "grad_norm": 9.161146143830141e-07,
      "learning_rate": 8.182413576596385e-07,
      "logits/chosen": 8.45948314666748,
      "logits/rejected": 8.418173789978027,
      "logps/chosen": -40.649803161621094,
      "logps/rejected": -42.114646911621094,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -40.649803161621094,
      "rewards/margins": 1.4648405313491821,
      "rewards/rejected": -42.114646911621094,
      "step": 1975
    },
    {
      "epoch": 1.0597089814350227,
      "grad_norm": 1.8078673772845872e-06,
      "learning_rate": 8.170386052085389e-07,
      "logits/chosen": 8.44881820678711,
      "logits/rejected": 8.045083045959473,
      "logps/chosen": -40.18886184692383,
      "logps/rejected": -40.51119613647461,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -40.18886184692383,
      "rewards/margins": 0.32233747839927673,
      "rewards/rejected": -40.51119613647461,
      "step": 1980
    },
    {
      "epoch": 1.0623850142164242,
      "grad_norm": 1.3394034110612003e-06,
      "learning_rate": 8.158327768475008e-07,
      "logits/chosen": 8.474889755249023,
      "logits/rejected": 8.058609962463379,
      "logps/chosen": -39.75669860839844,
      "logps/rejected": -41.23186111450195,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -39.75669860839844,
      "rewards/margins": 1.4751619100570679,
      "rewards/rejected": -41.23186111450195,
      "step": 1985
    },
    {
      "epoch": 1.0650610469978257,
      "grad_norm": 3.6865062913200944e-09,
      "learning_rate": 8.146238842754767e-07,
      "logits/chosen": 8.739977836608887,
      "logits/rejected": 8.270288467407227,
      "logps/chosen": -40.680946350097656,
      "logps/rejected": -41.545631408691406,
      "loss": 0.0,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -40.680946350097656,
      "rewards/margins": 0.8646896481513977,
      "rewards/rejected": -41.545631408691406,
      "step": 1990
    },
    {
      "epoch": 1.0677370797792274,
      "grad_norm": 7.802688627737534e-05,
      "learning_rate": 8.134119392211476e-07,
      "logits/chosen": 8.695432662963867,
      "logits/rejected": 8.185397148132324,
      "logps/chosen": -41.02822494506836,
      "logps/rejected": -41.93169021606445,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.02822494506836,
      "rewards/margins": 0.9034714698791504,
      "rewards/rejected": -41.93169021606445,
      "step": 1995
    },
    {
      "epoch": 1.0704131125606289,
      "grad_norm": 1.1757503810294671e-06,
      "learning_rate": 8.121969534428094e-07,
      "logits/chosen": 8.822378158569336,
      "logits/rejected": 8.411437034606934,
      "logps/chosen": -42.27022171020508,
      "logps/rejected": -43.45711135864258,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.27022171020508,
      "rewards/margins": 1.1868919134140015,
      "rewards/rejected": -43.45711135864258,
      "step": 2000
    },
    {
      "epoch": 1.0704131125606289,
      "eval_logits/chosen": 7.2450642585754395,
      "eval_logits/rejected": 7.007467746734619,
      "eval_logps/chosen": -40.77992248535156,
      "eval_logps/rejected": -42.01736831665039,
      "eval_loss": 3.1178174708657025e-07,
      "eval_rewards/accuracies": 0.5675074458122253,
      "eval_rewards/chosen": -40.77992248535156,
      "eval_rewards/margins": 1.2374484539031982,
      "eval_rewards/rejected": -42.01736831665039,
      "eval_runtime": 40.5107,
      "eval_samples_per_second": 33.201,
      "eval_steps_per_second": 8.319,
      "step": 2000
    },
    {
      "epoch": 1.0730891453420304,
      "grad_norm": 2.7941930189713664e-07,
      "learning_rate": 8.109789387282599e-07,
      "logits/chosen": 8.406800270080566,
      "logits/rejected": 8.302990913391113,
      "logps/chosen": -40.636985778808594,
      "logps/rejected": -41.70927047729492,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.636985778808594,
      "rewards/margins": 1.072289228439331,
      "rewards/rejected": -41.70927047729492,
      "step": 2005
    },
    {
      "epoch": 1.075765178123432,
      "grad_norm": 2.101473190733529e-11,
      "learning_rate": 8.097579068946827e-07,
      "logits/chosen": 8.42392349243164,
      "logits/rejected": 8.023496627807617,
      "logps/chosen": -40.87938690185547,
      "logps/rejected": -42.24049758911133,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -40.87938690185547,
      "rewards/margins": 1.3611103296279907,
      "rewards/rejected": -42.24049758911133,
      "step": 2010
    },
    {
      "epoch": 1.0784412109048336,
      "grad_norm": 3.5103260176475987e-09,
      "learning_rate": 8.085338697885344e-07,
      "logits/chosen": 8.387591361999512,
      "logits/rejected": 8.063955307006836,
      "logps/chosen": -41.28887176513672,
      "logps/rejected": -42.1496696472168,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.28887176513672,
      "rewards/margins": 0.8608001470565796,
      "rewards/rejected": -42.1496696472168,
      "step": 2015
    },
    {
      "epoch": 1.081117243686235,
      "grad_norm": 9.366365376165455e-10,
      "learning_rate": 8.073068392854282e-07,
      "logits/chosen": 9.011270523071289,
      "logits/rejected": 8.2319917678833,
      "logps/chosen": -40.543949127197266,
      "logps/rejected": -42.05414581298828,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -40.543949127197266,
      "rewards/margins": 1.5102001428604126,
      "rewards/rejected": -42.05414581298828,
      "step": 2020
    },
    {
      "epoch": 1.0837932764676368,
      "grad_norm": 1.7005169368649812e-05,
      "learning_rate": 8.060768272900193e-07,
      "logits/chosen": 9.145383834838867,
      "logits/rejected": 8.520723342895508,
      "logps/chosen": -42.4699821472168,
      "logps/rejected": -43.37511444091797,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.4699821472168,
      "rewards/margins": 0.9051315188407898,
      "rewards/rejected": -43.37511444091797,
      "step": 2025
    },
    {
      "epoch": 1.0864693092490383,
      "grad_norm": 4.195097076140448e-05,
      "learning_rate": 8.0484384573589e-07,
      "logits/chosen": 8.266286849975586,
      "logits/rejected": 8.424505233764648,
      "logps/chosen": -40.704959869384766,
      "logps/rejected": -41.65985870361328,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -40.704959869384766,
      "rewards/margins": 0.9548954963684082,
      "rewards/rejected": -41.65985870361328,
      "step": 2030
    },
    {
      "epoch": 1.0891453420304398,
      "grad_norm": 5.566721706753348e-05,
      "learning_rate": 8.03607906585432e-07,
      "logits/chosen": 8.620889663696289,
      "logits/rejected": 8.355611801147461,
      "logps/chosen": -41.4559211730957,
      "logps/rejected": -43.61249923706055,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.4559211730957,
      "rewards/margins": 2.1565775871276855,
      "rewards/rejected": -43.61249923706055,
      "step": 2035
    },
    {
      "epoch": 1.0918213748118415,
      "grad_norm": 3.321497541052285e-06,
      "learning_rate": 8.023690218297329e-07,
      "logits/chosen": 8.547624588012695,
      "logits/rejected": 8.592642784118652,
      "logps/chosen": -41.11884689331055,
      "logps/rejected": -42.3089714050293,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -41.11884689331055,
      "rewards/margins": 1.1901240348815918,
      "rewards/rejected": -42.3089714050293,
      "step": 2040
    },
    {
      "epoch": 1.094497407593243,
      "grad_norm": 1.6675678914805667e-07,
      "learning_rate": 8.01127203488458e-07,
      "logits/chosen": 8.132123947143555,
      "logits/rejected": 8.042007446289062,
      "logps/chosen": -39.87526321411133,
      "logps/rejected": -40.990150451660156,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -39.87526321411133,
      "rewards/margins": 1.1148872375488281,
      "rewards/rejected": -40.990150451660156,
      "step": 2045
    },
    {
      "epoch": 1.0971734403746445,
      "grad_norm": 4.163976797291173e-05,
      "learning_rate": 7.998824636097339e-07,
      "logits/chosen": 8.986612319946289,
      "logits/rejected": 8.556442260742188,
      "logps/chosen": -40.445838928222656,
      "logps/rejected": -42.195899963378906,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -40.445838928222656,
      "rewards/margins": 1.7500604391098022,
      "rewards/rejected": -42.195899963378906,
      "step": 2050
    },
    {
      "epoch": 1.0998494731560462,
      "grad_norm": 4.624883824363688e-09,
      "learning_rate": 7.986348142700328e-07,
      "logits/chosen": 8.554445266723633,
      "logits/rejected": 8.277101516723633,
      "logps/chosen": -40.31126403808594,
      "logps/rejected": -42.23910140991211,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.31126403808594,
      "rewards/margins": 1.9278380870819092,
      "rewards/rejected": -42.23910140991211,
      "step": 2055
    },
    {
      "epoch": 1.1025255059374477,
      "grad_norm": 1.546883033987733e-07,
      "learning_rate": 7.973842675740539e-07,
      "logits/chosen": 8.306285858154297,
      "logits/rejected": 8.320329666137695,
      "logps/chosen": -41.140647888183594,
      "logps/rejected": -41.80480194091797,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -41.140647888183594,
      "rewards/margins": 0.664159893989563,
      "rewards/rejected": -41.80480194091797,
      "step": 2060
    },
    {
      "epoch": 1.1052015387188494,
      "grad_norm": 4.034807895831652e-05,
      "learning_rate": 7.961308356546066e-07,
      "logits/chosen": 8.44710922241211,
      "logits/rejected": 8.226879119873047,
      "logps/chosen": -40.02400588989258,
      "logps/rejected": -41.54513168334961,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -40.02400588989258,
      "rewards/margins": 1.5211279392242432,
      "rewards/rejected": -41.54513168334961,
      "step": 2065
    },
    {
      "epoch": 1.107877571500251,
      "grad_norm": 4.578075163497712e-07,
      "learning_rate": 7.948745306724931e-07,
      "logits/chosen": 8.627179145812988,
      "logits/rejected": 8.316874504089355,
      "logps/chosen": -41.3073844909668,
      "logps/rejected": -42.705055236816406,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.3073844909668,
      "rewards/margins": 1.3976645469665527,
      "rewards/rejected": -42.705055236816406,
      "step": 2070
    },
    {
      "epoch": 1.1105536042816524,
      "grad_norm": 1.0250644810068454e-09,
      "learning_rate": 7.936153648163897e-07,
      "logits/chosen": 8.710594177246094,
      "logits/rejected": 8.54148006439209,
      "logps/chosen": -40.856868743896484,
      "logps/rejected": -42.92464828491211,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -40.856868743896484,
      "rewards/margins": 2.067783832550049,
      "rewards/rejected": -42.92464828491211,
      "step": 2075
    },
    {
      "epoch": 1.1132296370630541,
      "grad_norm": 6.890262997631695e-06,
      "learning_rate": 7.92353350302729e-07,
      "logits/chosen": 8.384446144104004,
      "logits/rejected": 8.116195678710938,
      "logps/chosen": -39.415672302246094,
      "logps/rejected": -40.97673416137695,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -39.415672302246094,
      "rewards/margins": 1.5610568523406982,
      "rewards/rejected": -40.97673416137695,
      "step": 2080
    },
    {
      "epoch": 1.1159056698444556,
      "grad_norm": 2.159244045559575e-07,
      "learning_rate": 7.910884993755816e-07,
      "logits/chosen": 8.64844036102295,
      "logits/rejected": 8.4146089553833,
      "logps/chosen": -40.42133712768555,
      "logps/rejected": -42.17142105102539,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -40.42133712768555,
      "rewards/margins": 1.7500842809677124,
      "rewards/rejected": -42.17142105102539,
      "step": 2085
    },
    {
      "epoch": 1.118581702625857,
      "grad_norm": 3.8231731613935785e-08,
      "learning_rate": 7.898208243065367e-07,
      "logits/chosen": 8.658584594726562,
      "logits/rejected": 8.730806350708008,
      "logps/chosen": -41.568092346191406,
      "logps/rejected": -41.944313049316406,
      "loss": 0.0001,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.568092346191406,
      "rewards/margins": 0.37622079253196716,
      "rewards/rejected": -41.944313049316406,
      "step": 2090
    },
    {
      "epoch": 1.1212577354072588,
      "grad_norm": 1.288588814136926e-08,
      "learning_rate": 7.88550337394583e-07,
      "logits/chosen": 8.935137748718262,
      "logits/rejected": 8.554451942443848,
      "logps/chosen": -40.218360900878906,
      "logps/rejected": -41.84646224975586,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.218360900878906,
      "rewards/margins": 1.6281015872955322,
      "rewards/rejected": -41.84646224975586,
      "step": 2095
    },
    {
      "epoch": 1.1239337681886603,
      "grad_norm": 3.438020051242254e-07,
      "learning_rate": 7.872770509659905e-07,
      "logits/chosen": 8.250994682312012,
      "logits/rejected": 8.14515495300293,
      "logps/chosen": -39.939308166503906,
      "logps/rejected": -41.26539611816406,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.939308166503906,
      "rewards/margins": 1.3260862827301025,
      "rewards/rejected": -41.26539611816406,
      "step": 2100
    },
    {
      "epoch": 1.1266098009700618,
      "grad_norm": 1.848211855951944e-05,
      "learning_rate": 7.860009773741896e-07,
      "logits/chosen": 8.618029594421387,
      "logits/rejected": 8.181163787841797,
      "logps/chosen": -40.06212615966797,
      "logps/rejected": -42.03260040283203,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.06212615966797,
      "rewards/margins": 1.9704763889312744,
      "rewards/rejected": -42.03260040283203,
      "step": 2105
    },
    {
      "epoch": 1.1292858337514635,
      "grad_norm": 1.4497593483416872e-06,
      "learning_rate": 7.84722128999652e-07,
      "logits/chosen": 8.6336088180542,
      "logits/rejected": 8.369260787963867,
      "logps/chosen": -40.98212432861328,
      "logps/rejected": -42.2159309387207,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.98212432861328,
      "rewards/margins": 1.233810544013977,
      "rewards/rejected": -42.2159309387207,
      "step": 2110
    },
    {
      "epoch": 1.131961866532865,
      "grad_norm": 0.0012040571520709415,
      "learning_rate": 7.834405182497699e-07,
      "logits/chosen": 8.384027481079102,
      "logits/rejected": 8.253840446472168,
      "logps/chosen": -40.520225524902344,
      "logps/rejected": -41.95452117919922,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -40.520225524902344,
      "rewards/margins": 1.4342975616455078,
      "rewards/rejected": -41.95452117919922,
      "step": 2115
    },
    {
      "epoch": 1.1346378993142665,
      "grad_norm": 4.745515723674222e-07,
      "learning_rate": 7.821561575587368e-07,
      "logits/chosen": 8.594862937927246,
      "logits/rejected": 8.447153091430664,
      "logps/chosen": -40.040931701660156,
      "logps/rejected": -41.00267791748047,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -40.040931701660156,
      "rewards/margins": 0.9617475271224976,
      "rewards/rejected": -41.00267791748047,
      "step": 2120
    },
    {
      "epoch": 1.1373139320956682,
      "grad_norm": 2.0375257881718297e-07,
      "learning_rate": 7.808690593874254e-07,
      "logits/chosen": 8.701242446899414,
      "logits/rejected": 8.27269172668457,
      "logps/chosen": -40.88854217529297,
      "logps/rejected": -41.848472595214844,
      "loss": 0.0018,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -40.88854217529297,
      "rewards/margins": 0.9599350690841675,
      "rewards/rejected": -41.848472595214844,
      "step": 2125
    },
    {
      "epoch": 1.1399899648770697,
      "grad_norm": 0.0002448884334940336,
      "learning_rate": 7.79579236223268e-07,
      "logits/chosen": 8.545909881591797,
      "logits/rejected": 7.890882968902588,
      "logps/chosen": -39.64358901977539,
      "logps/rejected": -42.72437286376953,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -39.64358901977539,
      "rewards/margins": 3.0807852745056152,
      "rewards/rejected": -42.72437286376953,
      "step": 2130
    },
    {
      "epoch": 1.1426659976584714,
      "grad_norm": 6.367315786181002e-07,
      "learning_rate": 7.782867005801346e-07,
      "logits/chosen": 8.717951774597168,
      "logits/rejected": 8.202997207641602,
      "logps/chosen": -39.86112976074219,
      "logps/rejected": -42.386024475097656,
      "loss": 0.0,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -39.86112976074219,
      "rewards/margins": 2.5248966217041016,
      "rewards/rejected": -42.386024475097656,
      "step": 2135
    },
    {
      "epoch": 1.145342030439873,
      "grad_norm": 7.71324902428603e-07,
      "learning_rate": 7.769914649982117e-07,
      "logits/chosen": 8.58655071258545,
      "logits/rejected": 8.170052528381348,
      "logps/chosen": -40.371742248535156,
      "logps/rejected": -42.495365142822266,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -40.371742248535156,
      "rewards/margins": 2.1236205101013184,
      "rewards/rejected": -42.495365142822266,
      "step": 2140
    },
    {
      "epoch": 1.1480180632212744,
      "grad_norm": 8.93364803440133e-09,
      "learning_rate": 7.756935420438803e-07,
      "logits/chosen": 8.457643508911133,
      "logits/rejected": 8.24944019317627,
      "logps/chosen": -40.57135772705078,
      "logps/rejected": -42.175621032714844,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -40.57135772705078,
      "rewards/margins": 1.604262113571167,
      "rewards/rejected": -42.175621032714844,
      "step": 2145
    },
    {
      "epoch": 1.1506940960026761,
      "grad_norm": 8.526416672830342e-05,
      "learning_rate": 7.743929443095951e-07,
      "logits/chosen": 8.606661796569824,
      "logits/rejected": 8.478116989135742,
      "logps/chosen": -41.13886260986328,
      "logps/rejected": -42.28718948364258,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.13886260986328,
      "rewards/margins": 1.1483221054077148,
      "rewards/rejected": -42.28718948364258,
      "step": 2150
    },
    {
      "epoch": 1.1533701287840776,
      "grad_norm": 4.4868601065516354e-05,
      "learning_rate": 7.730896844137609e-07,
      "logits/chosen": 8.28092098236084,
      "logits/rejected": 8.127330780029297,
      "logps/chosen": -40.70257568359375,
      "logps/rejected": -41.38616180419922,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -40.70257568359375,
      "rewards/margins": 0.6835864186286926,
      "rewards/rejected": -41.38616180419922,
      "step": 2155
    },
    {
      "epoch": 1.1560461615654791,
      "grad_norm": 1.517581487503091e-08,
      "learning_rate": 7.717837750006106e-07,
      "logits/chosen": 8.886369705200195,
      "logits/rejected": 8.697736740112305,
      "logps/chosen": -41.994773864746094,
      "logps/rejected": -42.7291374206543,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -41.994773864746094,
      "rewards/margins": 0.7343606948852539,
      "rewards/rejected": -42.7291374206543,
      "step": 2160
    },
    {
      "epoch": 1.1587221943468808,
      "grad_norm": 2.683699769021294e-08,
      "learning_rate": 7.704752287400832e-07,
      "logits/chosen": 8.586957931518555,
      "logits/rejected": 8.184276580810547,
      "logps/chosen": -41.64616775512695,
      "logps/rejected": -42.28727722167969,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -41.64616775512695,
      "rewards/margins": 0.6411095857620239,
      "rewards/rejected": -42.28727722167969,
      "step": 2165
    },
    {
      "epoch": 1.1613982271282823,
      "grad_norm": 7.520309082628661e-07,
      "learning_rate": 7.691640583277004e-07,
      "logits/chosen": 8.825390815734863,
      "logits/rejected": 8.263834953308105,
      "logps/chosen": -41.467002868652344,
      "logps/rejected": -43.19124984741211,
      "loss": 0.0,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -41.467002868652344,
      "rewards/margins": 1.7242462635040283,
      "rewards/rejected": -43.19124984741211,
      "step": 2170
    },
    {
      "epoch": 1.1640742599096838,
      "grad_norm": 2.507646332158615e-10,
      "learning_rate": 7.678502764844433e-07,
      "logits/chosen": 8.964905738830566,
      "logits/rejected": 8.28453254699707,
      "logps/chosen": -41.94441604614258,
      "logps/rejected": -42.04668045043945,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -41.94441604614258,
      "rewards/margins": 0.10226273536682129,
      "rewards/rejected": -42.04668045043945,
      "step": 2175
    },
    {
      "epoch": 1.1667502926910855,
      "grad_norm": 5.019293275196284e-06,
      "learning_rate": 7.665338959566288e-07,
      "logits/chosen": 8.768538475036621,
      "logits/rejected": 8.602264404296875,
      "logps/chosen": -41.781005859375,
      "logps/rejected": -42.875083923339844,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.781005859375,
      "rewards/margins": 1.0940748453140259,
      "rewards/rejected": -42.875083923339844,
      "step": 2180
    },
    {
      "epoch": 1.169426325472487,
      "grad_norm": 1.1070368520028168e-06,
      "learning_rate": 7.652149295157868e-07,
      "logits/chosen": 8.601483345031738,
      "logits/rejected": 8.105417251586914,
      "logps/chosen": -41.35640335083008,
      "logps/rejected": -41.93348693847656,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.35640335083008,
      "rewards/margins": 0.57708340883255,
      "rewards/rejected": -41.93348693847656,
      "step": 2185
    },
    {
      "epoch": 1.1721023582538885,
      "grad_norm": 0.00048313440939491784,
      "learning_rate": 7.638933899585354e-07,
      "logits/chosen": 8.284440994262695,
      "logits/rejected": 8.253382682800293,
      "logps/chosen": -40.96122360229492,
      "logps/rejected": -41.88115310668945,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -40.96122360229492,
      "rewards/margins": 0.9199298620223999,
      "rewards/rejected": -41.88115310668945,
      "step": 2190
    },
    {
      "epoch": 1.1747783910352902,
      "grad_norm": 0.041338124195654795,
      "learning_rate": 7.625692901064573e-07,
      "logits/chosen": 8.199808120727539,
      "logits/rejected": 8.141292572021484,
      "logps/chosen": -40.34364700317383,
      "logps/rejected": -41.13231658935547,
      "loss": 0.0001,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -40.34364700317383,
      "rewards/margins": 0.7886745929718018,
      "rewards/rejected": -41.13231658935547,
      "step": 2195
    },
    {
      "epoch": 1.1774544238166917,
      "grad_norm": 1.2096897115767505e-07,
      "learning_rate": 7.61242642805975e-07,
      "logits/chosen": 8.618353843688965,
      "logits/rejected": 8.716090202331543,
      "logps/chosen": -41.43954086303711,
      "logps/rejected": -42.253536224365234,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -41.43954086303711,
      "rewards/margins": 0.8139970898628235,
      "rewards/rejected": -42.253536224365234,
      "step": 2200
    },
    {
      "epoch": 1.1801304565980932,
      "grad_norm": 7.038101772689373e-08,
      "learning_rate": 7.599134609282266e-07,
      "logits/chosen": 8.902692794799805,
      "logits/rejected": 8.209165573120117,
      "logps/chosen": -41.19422149658203,
      "logps/rejected": -42.89899444580078,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.19422149658203,
      "rewards/margins": 1.7047698497772217,
      "rewards/rejected": -42.89899444580078,
      "step": 2205
    },
    {
      "epoch": 1.182806489379495,
      "grad_norm": 5.368342850833801e-09,
      "learning_rate": 7.585817573689402e-07,
      "logits/chosen": 8.607969284057617,
      "logits/rejected": 8.46239185333252,
      "logps/chosen": -40.59905242919922,
      "logps/rejected": -42.151268005371094,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -40.59905242919922,
      "rewards/margins": 1.5522193908691406,
      "rewards/rejected": -42.151268005371094,
      "step": 2210
    },
    {
      "epoch": 1.1854825221608964,
      "grad_norm": 5.233976859275776e-08,
      "learning_rate": 7.572475450483098e-07,
      "logits/chosen": 8.511218070983887,
      "logits/rejected": 8.185332298278809,
      "logps/chosen": -39.96785354614258,
      "logps/rejected": -41.31941604614258,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -39.96785354614258,
      "rewards/margins": 1.3515615463256836,
      "rewards/rejected": -41.31941604614258,
      "step": 2215
    },
    {
      "epoch": 1.188158554942298,
      "grad_norm": 2.6535416025913438e-06,
      "learning_rate": 7.559108369108689e-07,
      "logits/chosen": 8.501649856567383,
      "logits/rejected": 8.205302238464355,
      "logps/chosen": -39.40612030029297,
      "logps/rejected": -40.170108795166016,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -39.40612030029297,
      "rewards/margins": 0.7639927864074707,
      "rewards/rejected": -40.170108795166016,
      "step": 2220
    },
    {
      "epoch": 1.1908345877236997,
      "grad_norm": 2.076636675531797e-07,
      "learning_rate": 7.54571645925366e-07,
      "logits/chosen": 8.915776252746582,
      "logits/rejected": 8.282099723815918,
      "logps/chosen": -39.499900817871094,
      "logps/rejected": -42.26732635498047,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -39.499900817871094,
      "rewards/margins": 2.7674288749694824,
      "rewards/rejected": -42.26732635498047,
      "step": 2225
    },
    {
      "epoch": 1.1935106205051011,
      "grad_norm": 1.770966811824327e-08,
      "learning_rate": 7.532299850846378e-07,
      "logits/chosen": 8.668721199035645,
      "logits/rejected": 8.15053653717041,
      "logps/chosen": -40.732810974121094,
      "logps/rejected": -42.174766540527344,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -40.732810974121094,
      "rewards/margins": 1.441957712173462,
      "rewards/rejected": -42.174766540527344,
      "step": 2230
    },
    {
      "epoch": 1.1961866532865026,
      "grad_norm": 1.1293539338067661e-07,
      "learning_rate": 7.518858674054838e-07,
      "logits/chosen": 8.647680282592773,
      "logits/rejected": 8.115179061889648,
      "logps/chosen": -40.324615478515625,
      "logps/rejected": -41.052040100097656,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -40.324615478515625,
      "rewards/margins": 0.7274282574653625,
      "rewards/rejected": -41.052040100097656,
      "step": 2235
    },
    {
      "epoch": 1.1988626860679044,
      "grad_norm": 1.774380533255003e-10,
      "learning_rate": 7.505393059285394e-07,
      "logits/chosen": 8.428884506225586,
      "logits/rejected": 8.079434394836426,
      "logps/chosen": -40.62153625488281,
      "logps/rejected": -41.99386978149414,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -40.62153625488281,
      "rewards/margins": 1.3723335266113281,
      "rewards/rejected": -41.99386978149414,
      "step": 2240
    },
    {
      "epoch": 1.2015387188493059,
      "grad_norm": 7.881354785945644e-06,
      "learning_rate": 7.491903137181501e-07,
      "logits/chosen": 8.486298561096191,
      "logits/rejected": 8.506063461303711,
      "logps/chosen": -40.976165771484375,
      "logps/rejected": -42.27082443237305,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -40.976165771484375,
      "rewards/margins": 1.2946627140045166,
      "rewards/rejected": -42.27082443237305,
      "step": 2245
    },
    {
      "epoch": 1.2042147516307076,
      "grad_norm": 2.513369944743648e-07,
      "learning_rate": 7.478389038622441e-07,
      "logits/chosen": 8.356260299682617,
      "logits/rejected": 8.329123497009277,
      "logps/chosen": -40.45479202270508,
      "logps/rejected": -41.39447784423828,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -40.45479202270508,
      "rewards/margins": 0.9396904110908508,
      "rewards/rejected": -41.39447784423828,
      "step": 2250
    },
    {
      "epoch": 1.206890784412109,
      "grad_norm": 0.0005368014391819825,
      "learning_rate": 7.46485089472206e-07,
      "logits/chosen": 8.505821228027344,
      "logits/rejected": 8.476261138916016,
      "logps/chosen": -41.28764724731445,
      "logps/rejected": -42.163978576660156,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -41.28764724731445,
      "rewards/margins": 0.8763296008110046,
      "rewards/rejected": -42.163978576660156,
      "step": 2255
    },
    {
      "epoch": 1.2095668171935106,
      "grad_norm": 0.00011978943869628825,
      "learning_rate": 7.451288836827487e-07,
      "logits/chosen": 8.45439624786377,
      "logits/rejected": 8.619742393493652,
      "logps/chosen": -41.024810791015625,
      "logps/rejected": -41.434513092041016,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -41.024810791015625,
      "rewards/margins": 0.40970200300216675,
      "rewards/rejected": -41.434513092041016,
      "step": 2260
    },
    {
      "epoch": 1.2122428499749123,
      "grad_norm": 2.758227806494229e-10,
      "learning_rate": 7.437702996517869e-07,
      "logits/chosen": 8.706487655639648,
      "logits/rejected": 8.562585830688477,
      "logps/chosen": -39.77735137939453,
      "logps/rejected": -41.421119689941406,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -39.77735137939453,
      "rewards/margins": 1.6437727212905884,
      "rewards/rejected": -41.421119689941406,
      "step": 2265
    },
    {
      "epoch": 1.2149188827563138,
      "grad_norm": 1.1688229226088774e-07,
      "learning_rate": 7.424093505603087e-07,
      "logits/chosen": 8.765771865844727,
      "logits/rejected": 8.405631065368652,
      "logps/chosen": -39.97887420654297,
      "logps/rejected": -41.925376892089844,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -39.97887420654297,
      "rewards/margins": 1.9465036392211914,
      "rewards/rejected": -41.925376892089844,
      "step": 2270
    },
    {
      "epoch": 1.2175949155377153,
      "grad_norm": 1.902147326128656e-06,
      "learning_rate": 7.410460496122482e-07,
      "logits/chosen": 8.554079055786133,
      "logits/rejected": 8.12889575958252,
      "logps/chosen": -41.50530242919922,
      "logps/rejected": -42.36071014404297,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.50530242919922,
      "rewards/margins": 0.8554102182388306,
      "rewards/rejected": -42.36071014404297,
      "step": 2275
    },
    {
      "epoch": 1.220270948319117,
      "grad_norm": 0.00017505075350152178,
      "learning_rate": 7.396804100343572e-07,
      "logits/chosen": 8.910680770874023,
      "logits/rejected": 8.220748901367188,
      "logps/chosen": -40.28874588012695,
      "logps/rejected": -41.78064727783203,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -40.28874588012695,
      "rewards/margins": 1.4918994903564453,
      "rewards/rejected": -41.78064727783203,
      "step": 2280
    },
    {
      "epoch": 1.2229469811005185,
      "grad_norm": 0.00021053339995770096,
      "learning_rate": 7.383124450760768e-07,
      "logits/chosen": 8.962723731994629,
      "logits/rejected": 8.456936836242676,
      "logps/chosen": -40.37126922607422,
      "logps/rejected": -42.131126403808594,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.37126922607422,
      "rewards/margins": 1.7598581314086914,
      "rewards/rejected": -42.131126403808594,
      "step": 2285
    },
    {
      "epoch": 1.22562301388192,
      "grad_norm": 4.1889449556908217e-10,
      "learning_rate": 7.369421680094091e-07,
      "logits/chosen": 9.063736915588379,
      "logits/rejected": 8.632521629333496,
      "logps/chosen": -40.83104705810547,
      "logps/rejected": -42.700775146484375,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -40.83104705810547,
      "rewards/margins": 1.8697303533554077,
      "rewards/rejected": -42.700775146484375,
      "step": 2290
    },
    {
      "epoch": 1.2282990466633217,
      "grad_norm": 2.806326530729962e-05,
      "learning_rate": 7.355695921287881e-07,
      "logits/chosen": 8.770237922668457,
      "logits/rejected": 8.363417625427246,
      "logps/chosen": -40.73759078979492,
      "logps/rejected": -41.025428771972656,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -40.73759078979492,
      "rewards/margins": 0.28783732652664185,
      "rewards/rejected": -41.025428771972656,
      "step": 2295
    },
    {
      "epoch": 1.2309750794447232,
      "grad_norm": 8.513334799377988e-08,
      "learning_rate": 7.341947307509513e-07,
      "logits/chosen": 8.68481731414795,
      "logits/rejected": 8.428956985473633,
      "logps/chosen": -40.888038635253906,
      "logps/rejected": -42.52493667602539,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -40.888038635253906,
      "rewards/margins": 1.6368995904922485,
      "rewards/rejected": -42.52493667602539,
      "step": 2300
    },
    {
      "epoch": 1.233651112226125,
      "grad_norm": 1.824512679842653e-10,
      "learning_rate": 7.328175972148094e-07,
      "logits/chosen": 8.70347785949707,
      "logits/rejected": 8.293722152709961,
      "logps/chosen": -40.107975006103516,
      "logps/rejected": -42.11811447143555,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -40.107975006103516,
      "rewards/margins": 2.0101377964019775,
      "rewards/rejected": -42.11811447143555,
      "step": 2305
    },
    {
      "epoch": 1.2363271450075264,
      "grad_norm": 2.017785882524667e-10,
      "learning_rate": 7.314382048813185e-07,
      "logits/chosen": 8.495558738708496,
      "logits/rejected": 7.869775295257568,
      "logps/chosen": -40.17514419555664,
      "logps/rejected": -43.199771881103516,
      "loss": 0.0003,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -40.17514419555664,
      "rewards/margins": 3.024625062942505,
      "rewards/rejected": -43.199771881103516,
      "step": 2310
    },
    {
      "epoch": 1.2390031777889279,
      "grad_norm": 4.198501928119521e-08,
      "learning_rate": 7.300565671333486e-07,
      "logits/chosen": 8.782133102416992,
      "logits/rejected": 8.277894973754883,
      "logps/chosen": -40.93571853637695,
      "logps/rejected": -42.298927307128906,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -40.93571853637695,
      "rewards/margins": 1.3632045984268188,
      "rewards/rejected": -42.298927307128906,
      "step": 2315
    },
    {
      "epoch": 1.2416792105703296,
      "grad_norm": 2.6340808748596537e-05,
      "learning_rate": 7.286726973755554e-07,
      "logits/chosen": 8.759393692016602,
      "logits/rejected": 8.505353927612305,
      "logps/chosen": -41.74432373046875,
      "logps/rejected": -41.89595413208008,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.74432373046875,
      "rewards/margins": 0.1516321450471878,
      "rewards/rejected": -41.89595413208008,
      "step": 2320
    },
    {
      "epoch": 1.244355243351731,
      "grad_norm": 2.483248990420863e-05,
      "learning_rate": 7.272866090342493e-07,
      "logits/chosen": 8.273099899291992,
      "logits/rejected": 8.017354965209961,
      "logps/chosen": -39.90392303466797,
      "logps/rejected": -41.7261848449707,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -39.90392303466797,
      "rewards/margins": 1.8222602605819702,
      "rewards/rejected": -41.7261848449707,
      "step": 2325
    },
    {
      "epoch": 1.2470312761331326,
      "grad_norm": 5.264857678984263e-07,
      "learning_rate": 7.258983155572656e-07,
      "logits/chosen": 8.998651504516602,
      "logits/rejected": 8.72075366973877,
      "logps/chosen": -42.4549560546875,
      "logps/rejected": -42.265193939208984,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.4549560546875,
      "rewards/margins": -0.18975695967674255,
      "rewards/rejected": -42.265193939208984,
      "step": 2330
    },
    {
      "epoch": 1.2497073089145343,
      "grad_norm": 7.337966513512116e-05,
      "learning_rate": 7.245078304138335e-07,
      "logits/chosen": 8.475242614746094,
      "logits/rejected": 8.307760238647461,
      "logps/chosen": -40.506065368652344,
      "logps/rejected": -41.600563049316406,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -40.506065368652344,
      "rewards/margins": 1.0944931507110596,
      "rewards/rejected": -41.600563049316406,
      "step": 2335
    },
    {
      "epoch": 1.2523833416959358,
      "grad_norm": 3.5507335338129486e-09,
      "learning_rate": 7.231151670944462e-07,
      "logits/chosen": 8.902246475219727,
      "logits/rejected": 8.067272186279297,
      "logps/chosen": -41.660545349121094,
      "logps/rejected": -42.258522033691406,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -41.660545349121094,
      "rewards/margins": 0.597974419593811,
      "rewards/rejected": -42.258522033691406,
      "step": 2340
    },
    {
      "epoch": 1.2550593744773373,
      "grad_norm": 3.189400627470942e-05,
      "learning_rate": 7.217203391107291e-07,
      "logits/chosen": 8.499944686889648,
      "logits/rejected": 8.12531852722168,
      "logps/chosen": -41.47569274902344,
      "logps/rejected": -42.175228118896484,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.47569274902344,
      "rewards/margins": 0.6995365619659424,
      "rewards/rejected": -42.175228118896484,
      "step": 2345
    },
    {
      "epoch": 1.257735407258739,
      "grad_norm": 0.29839958899710833,
      "learning_rate": 7.203233599953096e-07,
      "logits/chosen": 8.346668243408203,
      "logits/rejected": 8.014691352844238,
      "logps/chosen": -39.29924774169922,
      "logps/rejected": -40.19755935668945,
      "loss": 0.0006,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -39.29924774169922,
      "rewards/margins": 0.8983109593391418,
      "rewards/rejected": -40.19755935668945,
      "step": 2350
    },
    {
      "epoch": 1.2604114400401405,
      "grad_norm": 5.266689759473824e-05,
      "learning_rate": 7.189242433016852e-07,
      "logits/chosen": 8.41518497467041,
      "logits/rejected": 8.157347679138184,
      "logps/chosen": -41.577552795410156,
      "logps/rejected": -43.21418762207031,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.577552795410156,
      "rewards/margins": 1.6366382837295532,
      "rewards/rejected": -43.21418762207031,
      "step": 2355
    },
    {
      "epoch": 1.263087472821542,
      "grad_norm": 2.7561907247547575e-11,
      "learning_rate": 7.17523002604092e-07,
      "logits/chosen": 8.851011276245117,
      "logits/rejected": 8.375604629516602,
      "logps/chosen": -41.504669189453125,
      "logps/rejected": -42.88933181762695,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.504669189453125,
      "rewards/margins": 1.3846628665924072,
      "rewards/rejected": -42.88933181762695,
      "step": 2360
    },
    {
      "epoch": 1.2657635056029437,
      "grad_norm": 4.218024672871632e-07,
      "learning_rate": 7.161196514973734e-07,
      "logits/chosen": 8.617021560668945,
      "logits/rejected": 8.004721641540527,
      "logps/chosen": -40.308563232421875,
      "logps/rejected": -42.25050354003906,
      "loss": 0.0017,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -40.308563232421875,
      "rewards/margins": 1.941937804222107,
      "rewards/rejected": -42.25050354003906,
      "step": 2365
    },
    {
      "epoch": 1.2684395383843452,
      "grad_norm": 4.037749140875123e-07,
      "learning_rate": 7.147142035968483e-07,
      "logits/chosen": 8.663287162780762,
      "logits/rejected": 8.161401748657227,
      "logps/chosen": -40.75321960449219,
      "logps/rejected": -42.39347457885742,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -40.75321960449219,
      "rewards/margins": 1.6402534246444702,
      "rewards/rejected": -42.39347457885742,
      "step": 2370
    },
    {
      "epoch": 1.2711155711657467,
      "grad_norm": 6.167313201711084e-06,
      "learning_rate": 7.133066725381781e-07,
      "logits/chosen": 8.902474403381348,
      "logits/rejected": 8.259099006652832,
      "logps/chosen": -41.818199157714844,
      "logps/rejected": -42.90471649169922,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -41.818199157714844,
      "rewards/margins": 1.0865098237991333,
      "rewards/rejected": -42.90471649169922,
      "step": 2375
    },
    {
      "epoch": 1.2737916039471484,
      "grad_norm": 9.884818779452493e-08,
      "learning_rate": 7.118970719772354e-07,
      "logits/chosen": 8.613125801086426,
      "logits/rejected": 8.22067928314209,
      "logps/chosen": -40.86578369140625,
      "logps/rejected": -42.80434799194336,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -40.86578369140625,
      "rewards/margins": 1.9385671615600586,
      "rewards/rejected": -42.80434799194336,
      "step": 2380
    },
    {
      "epoch": 1.27646763672855,
      "grad_norm": 0.0002653684648249868,
      "learning_rate": 7.104854155899711e-07,
      "logits/chosen": 8.731677055358887,
      "logits/rejected": 8.44113540649414,
      "logps/chosen": -42.01833724975586,
      "logps/rejected": -42.770538330078125,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -42.01833724975586,
      "rewards/margins": 0.7522023916244507,
      "rewards/rejected": -42.770538330078125,
      "step": 2385
    },
    {
      "epoch": 1.2791436695099514,
      "grad_norm": 1.4042740382718883e-11,
      "learning_rate": 7.090717170722817e-07,
      "logits/chosen": 8.50440502166748,
      "logits/rejected": 8.578502655029297,
      "logps/chosen": -42.388492584228516,
      "logps/rejected": -43.734596252441406,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -42.388492584228516,
      "rewards/margins": 1.346106767654419,
      "rewards/rejected": -43.734596252441406,
      "step": 2390
    },
    {
      "epoch": 1.2818197022913531,
      "grad_norm": 9.263269870484311e-06,
      "learning_rate": 7.076559901398762e-07,
      "logits/chosen": 9.335229873657227,
      "logits/rejected": 8.712800979614258,
      "logps/chosen": -40.85382080078125,
      "logps/rejected": -43.03199005126953,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -40.85382080078125,
      "rewards/margins": 2.1781651973724365,
      "rewards/rejected": -43.03199005126953,
      "step": 2395
    },
    {
      "epoch": 1.2844957350727546,
      "grad_norm": 1.5843446801085798e-05,
      "learning_rate": 7.062382485281436e-07,
      "logits/chosen": 8.836427688598633,
      "logits/rejected": 8.298908233642578,
      "logps/chosen": -40.521949768066406,
      "logps/rejected": -41.85470199584961,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -40.521949768066406,
      "rewards/margins": 1.332749605178833,
      "rewards/rejected": -41.85470199584961,
      "step": 2400
    },
    {
      "epoch": 1.2844957350727546,
      "eval_logits/chosen": 7.18833589553833,
      "eval_logits/rejected": 6.942528247833252,
      "eval_logps/chosen": -40.9809455871582,
      "eval_logps/rejected": -42.20896530151367,
      "eval_loss": 3.199867535386147e-07,
      "eval_rewards/accuracies": 0.5645400881767273,
      "eval_rewards/chosen": -40.9809455871582,
      "eval_rewards/margins": 1.2280218601226807,
      "eval_rewards/rejected": -42.20896530151367,
      "eval_runtime": 40.518,
      "eval_samples_per_second": 33.195,
      "eval_steps_per_second": 8.317,
      "step": 2400
    },
    {
      "epoch": 1.287171767854156,
      "grad_norm": 1.571179791147905e-10,
      "learning_rate": 7.048185059920193e-07,
      "logits/chosen": 8.714685440063477,
      "logits/rejected": 8.403592109680176,
      "logps/chosen": -41.647098541259766,
      "logps/rejected": -43.28430938720703,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -41.647098541259766,
      "rewards/margins": 1.6372133493423462,
      "rewards/rejected": -43.28430938720703,
      "step": 2405
    },
    {
      "epoch": 1.2898478006355578,
      "grad_norm": 3.0588356424467986e-07,
      "learning_rate": 7.033967763058516e-07,
      "logits/chosen": 8.547478675842285,
      "logits/rejected": 8.087888717651367,
      "logps/chosen": -40.34328079223633,
      "logps/rejected": -41.562259674072266,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -40.34328079223633,
      "rewards/margins": 1.2189751863479614,
      "rewards/rejected": -41.562259674072266,
      "step": 2410
    },
    {
      "epoch": 1.2925238334169593,
      "grad_norm": 3.7050150197833705e-11,
      "learning_rate": 7.019730732632681e-07,
      "logits/chosen": 8.655824661254883,
      "logits/rejected": 8.174910545349121,
      "logps/chosen": -41.71727752685547,
      "logps/rejected": -42.924339294433594,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.71727752685547,
      "rewards/margins": 1.2070648670196533,
      "rewards/rejected": -42.924339294433594,
      "step": 2415
    },
    {
      "epoch": 1.2951998661983608,
      "grad_norm": 8.798761144920488e-09,
      "learning_rate": 7.005474106770418e-07,
      "logits/chosen": 8.676004409790039,
      "logits/rejected": 8.410015106201172,
      "logps/chosen": -41.435325622558594,
      "logps/rejected": -42.68471145629883,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.435325622558594,
      "rewards/margins": 1.2493826150894165,
      "rewards/rejected": -42.68471145629883,
      "step": 2420
    },
    {
      "epoch": 1.2978758989797625,
      "grad_norm": 6.038303646700376e-08,
      "learning_rate": 6.991198023789577e-07,
      "logits/chosen": 8.771553039550781,
      "logits/rejected": 8.683374404907227,
      "logps/chosen": -42.4216423034668,
      "logps/rejected": -42.75839614868164,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -42.4216423034668,
      "rewards/margins": 0.3367560803890228,
      "rewards/rejected": -42.75839614868164,
      "step": 2425
    },
    {
      "epoch": 1.300551931761164,
      "grad_norm": 7.260274701556586e-09,
      "learning_rate": 6.976902622196776e-07,
      "logits/chosen": 8.623698234558105,
      "logits/rejected": 8.419386863708496,
      "logps/chosen": -41.76908493041992,
      "logps/rejected": -42.57697296142578,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -41.76908493041992,
      "rewards/margins": 0.8078799247741699,
      "rewards/rejected": -42.57697296142578,
      "step": 2430
    },
    {
      "epoch": 1.3032279645425655,
      "grad_norm": 2.7354807323703488e-11,
      "learning_rate": 6.962588040686064e-07,
      "logits/chosen": 8.827204704284668,
      "logits/rejected": 8.322413444519043,
      "logps/chosen": -41.5187873840332,
      "logps/rejected": -42.699493408203125,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.5187873840332,
      "rewards/margins": 1.1807054281234741,
      "rewards/rejected": -42.699493408203125,
      "step": 2435
    },
    {
      "epoch": 1.3059039973239672,
      "grad_norm": 2.6773791010464444e-10,
      "learning_rate": 6.948254418137573e-07,
      "logits/chosen": 8.62433910369873,
      "logits/rejected": 8.21703052520752,
      "logps/chosen": -41.080535888671875,
      "logps/rejected": -42.85414123535156,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -41.080535888671875,
      "rewards/margins": 1.7736066579818726,
      "rewards/rejected": -42.85414123535156,
      "step": 2440
    },
    {
      "epoch": 1.3085800301053687,
      "grad_norm": 0.006580855672145723,
      "learning_rate": 6.933901893616174e-07,
      "logits/chosen": 8.711406707763672,
      "logits/rejected": 8.392984390258789,
      "logps/chosen": -40.449989318847656,
      "logps/rejected": -41.890872955322266,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -40.449989318847656,
      "rewards/margins": 1.4408788681030273,
      "rewards/rejected": -41.890872955322266,
      "step": 2445
    },
    {
      "epoch": 1.3112560628867704,
      "grad_norm": 4.3504241809224855e-08,
      "learning_rate": 6.919530606370121e-07,
      "logits/chosen": 8.752779006958008,
      "logits/rejected": 8.183549880981445,
      "logps/chosen": -42.184120178222656,
      "logps/rejected": -42.82275390625,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.184120178222656,
      "rewards/margins": 0.6386359333992004,
      "rewards/rejected": -42.82275390625,
      "step": 2450
    },
    {
      "epoch": 1.313932095668172,
      "grad_norm": 8.266462065946908e-09,
      "learning_rate": 6.905140695829706e-07,
      "logits/chosen": 8.778003692626953,
      "logits/rejected": 8.015851020812988,
      "logps/chosen": -41.44707107543945,
      "logps/rejected": -43.082847595214844,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -41.44707107543945,
      "rewards/margins": 1.6357781887054443,
      "rewards/rejected": -43.082847595214844,
      "step": 2455
    },
    {
      "epoch": 1.3166081284495736,
      "grad_norm": 1.754618605966783e-13,
      "learning_rate": 6.890732301605904e-07,
      "logits/chosen": 8.421499252319336,
      "logits/rejected": 8.113317489624023,
      "logps/chosen": -42.11381149291992,
      "logps/rejected": -42.73085403442383,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.11381149291992,
      "rewards/margins": 0.6170390844345093,
      "rewards/rejected": -42.73085403442383,
      "step": 2460
    },
    {
      "epoch": 1.3192841612309751,
      "grad_norm": 1.4550976081624315e-08,
      "learning_rate": 6.876305563489021e-07,
      "logits/chosen": 8.965174674987793,
      "logits/rejected": 8.695558547973633,
      "logps/chosen": -40.9244384765625,
      "logps/rejected": -42.84111785888672,
      "loss": 0.0,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -40.9244384765625,
      "rewards/margins": 1.9166736602783203,
      "rewards/rejected": -42.84111785888672,
      "step": 2465
    },
    {
      "epoch": 1.3219601940123766,
      "grad_norm": 4.211747862621378e-06,
      "learning_rate": 6.861860621447331e-07,
      "logits/chosen": 9.004940032958984,
      "logits/rejected": 8.56294059753418,
      "logps/chosen": -41.2769660949707,
      "logps/rejected": -41.99330520629883,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -41.2769660949707,
      "rewards/margins": 0.7163404226303101,
      "rewards/rejected": -41.99330520629883,
      "step": 2470
    },
    {
      "epoch": 1.3246362267937783,
      "grad_norm": 1.3576926471003525e-07,
      "learning_rate": 6.847397615625725e-07,
      "logits/chosen": 8.54549503326416,
      "logits/rejected": 8.454249382019043,
      "logps/chosen": -42.046653747558594,
      "logps/rejected": -42.49961471557617,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.046653747558594,
      "rewards/margins": 0.45295557379722595,
      "rewards/rejected": -42.49961471557617,
      "step": 2475
    },
    {
      "epoch": 1.3273122595751798,
      "grad_norm": 1.749330501273377e-07,
      "learning_rate": 6.83291668634435e-07,
      "logits/chosen": 9.07643985748291,
      "logits/rejected": 8.617594718933105,
      "logps/chosen": -40.69664764404297,
      "logps/rejected": -43.09975051879883,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -40.69664764404297,
      "rewards/margins": 2.4031033515930176,
      "rewards/rejected": -43.09975051879883,
      "step": 2480
    },
    {
      "epoch": 1.3299882923565813,
      "grad_norm": 0.0005062999094676158,
      "learning_rate": 6.818417974097246e-07,
      "logits/chosen": 8.428836822509766,
      "logits/rejected": 8.086463928222656,
      "logps/chosen": -40.843238830566406,
      "logps/rejected": -41.650115966796875,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -40.843238830566406,
      "rewards/margins": 0.8068798184394836,
      "rewards/rejected": -41.650115966796875,
      "step": 2485
    },
    {
      "epoch": 1.332664325137983,
      "grad_norm": 3.2960091028267025e-07,
      "learning_rate": 6.803901619550981e-07,
      "logits/chosen": 8.54697322845459,
      "logits/rejected": 8.415047645568848,
      "logps/chosen": -40.57067108154297,
      "logps/rejected": -41.406795501708984,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -40.57067108154297,
      "rewards/margins": 0.8361251950263977,
      "rewards/rejected": -41.406795501708984,
      "step": 2490
    },
    {
      "epoch": 1.3353403579193845,
      "grad_norm": 2.9652714309888683e-06,
      "learning_rate": 6.789367763543292e-07,
      "logits/chosen": 8.747296333312988,
      "logits/rejected": 8.594714164733887,
      "logps/chosen": -42.303531646728516,
      "logps/rejected": -42.46321487426758,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -42.303531646728516,
      "rewards/margins": 0.15968793630599976,
      "rewards/rejected": -42.46321487426758,
      "step": 2495
    },
    {
      "epoch": 1.338016390700786,
      "grad_norm": 4.1589347068855514e-07,
      "learning_rate": 6.774816547081714e-07,
      "logits/chosen": 8.406895637512207,
      "logits/rejected": 7.916224479675293,
      "logps/chosen": -40.95653533935547,
      "logps/rejected": -42.47511672973633,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -40.95653533935547,
      "rewards/margins": 1.518578290939331,
      "rewards/rejected": -42.47511672973633,
      "step": 2500
    },
    {
      "epoch": 1.3406924234821878,
      "grad_norm": 8.431653731678739e-08,
      "learning_rate": 6.760248111342211e-07,
      "logits/chosen": 8.756312370300293,
      "logits/rejected": 8.310503959655762,
      "logps/chosen": -42.473899841308594,
      "logps/rejected": -44.119346618652344,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.473899841308594,
      "rewards/margins": 1.6454436779022217,
      "rewards/rejected": -44.119346618652344,
      "step": 2505
    },
    {
      "epoch": 1.3433684562635893,
      "grad_norm": 0.0020758887081444408,
      "learning_rate": 6.745662597667813e-07,
      "logits/chosen": 8.71196174621582,
      "logits/rejected": 8.32054615020752,
      "logps/chosen": -41.26520919799805,
      "logps/rejected": -42.23027038574219,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.26520919799805,
      "rewards/margins": 0.9650588035583496,
      "rewards/rejected": -42.23027038574219,
      "step": 2510
    },
    {
      "epoch": 1.3460444890449907,
      "grad_norm": 5.018287026327811e-10,
      "learning_rate": 6.731060147567236e-07,
      "logits/chosen": 8.812787055969238,
      "logits/rejected": 8.561102867126465,
      "logps/chosen": -41.989990234375,
      "logps/rejected": -43.05455780029297,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -41.989990234375,
      "rewards/margins": 1.0645701885223389,
      "rewards/rejected": -43.05455780029297,
      "step": 2515
    },
    {
      "epoch": 1.3487205218263925,
      "grad_norm": 2.0039794717480037e-09,
      "learning_rate": 6.716440902713515e-07,
      "logits/chosen": 8.890619277954102,
      "logits/rejected": 8.662046432495117,
      "logps/chosen": -41.35208511352539,
      "logps/rejected": -42.011566162109375,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.35208511352539,
      "rewards/margins": 0.6594769954681396,
      "rewards/rejected": -42.011566162109375,
      "step": 2520
    },
    {
      "epoch": 1.351396554607794,
      "grad_norm": 1.0519214631288001e-06,
      "learning_rate": 6.701805004942627e-07,
      "logits/chosen": 8.953778266906738,
      "logits/rejected": 8.594943046569824,
      "logps/chosen": -41.30754852294922,
      "logps/rejected": -42.41514205932617,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.30754852294922,
      "rewards/margins": 1.1075888872146606,
      "rewards/rejected": -42.41514205932617,
      "step": 2525
    },
    {
      "epoch": 1.3540725873891954,
      "grad_norm": 7.248119894564718e-07,
      "learning_rate": 6.687152596252119e-07,
      "logits/chosen": 8.74620532989502,
      "logits/rejected": 8.61555004119873,
      "logps/chosen": -41.74907684326172,
      "logps/rejected": -42.83419418334961,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -41.74907684326172,
      "rewards/margins": 1.085118055343628,
      "rewards/rejected": -42.83419418334961,
      "step": 2530
    },
    {
      "epoch": 1.3567486201705972,
      "grad_norm": 6.083875027474749e-10,
      "learning_rate": 6.672483818799722e-07,
      "logits/chosen": 8.921418190002441,
      "logits/rejected": 8.451409339904785,
      "logps/chosen": -41.394901275634766,
      "logps/rejected": -42.724754333496094,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -41.394901275634766,
      "rewards/margins": 1.3298547267913818,
      "rewards/rejected": -42.724754333496094,
      "step": 2535
    },
    {
      "epoch": 1.3594246529519987,
      "grad_norm": 5.15472179116309e-10,
      "learning_rate": 6.657798814901978e-07,
      "logits/chosen": 8.760702133178711,
      "logits/rejected": 8.053281784057617,
      "logps/chosen": -42.08037185668945,
      "logps/rejected": -42.4610710144043,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -42.08037185668945,
      "rewards/margins": 0.38069939613342285,
      "rewards/rejected": -42.4610710144043,
      "step": 2540
    },
    {
      "epoch": 1.3621006857334002,
      "grad_norm": 4.235811601808339e-06,
      "learning_rate": 6.643097727032863e-07,
      "logits/chosen": 8.981941223144531,
      "logits/rejected": 8.411722183227539,
      "logps/chosen": -42.244911193847656,
      "logps/rejected": -43.563201904296875,
      "loss": 0.0003,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.244911193847656,
      "rewards/margins": 1.3182884454727173,
      "rewards/rejected": -43.563201904296875,
      "step": 2545
    },
    {
      "epoch": 1.3647767185148019,
      "grad_norm": 1.0000092335168433e-06,
      "learning_rate": 6.628380697822392e-07,
      "logits/chosen": 9.122479438781738,
      "logits/rejected": 8.642129898071289,
      "logps/chosen": -41.744998931884766,
      "logps/rejected": -42.602012634277344,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -41.744998931884766,
      "rewards/margins": 0.8570182919502258,
      "rewards/rejected": -42.602012634277344,
      "step": 2550
    },
    {
      "epoch": 1.3674527512962034,
      "grad_norm": 3.3407544871507295e-05,
      "learning_rate": 6.61364787005525e-07,
      "logits/chosen": 8.453834533691406,
      "logits/rejected": 8.173998832702637,
      "logps/chosen": -40.599891662597656,
      "logps/rejected": -42.157222747802734,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -40.599891662597656,
      "rewards/margins": 1.5573351383209229,
      "rewards/rejected": -42.157222747802734,
      "step": 2555
    },
    {
      "epoch": 1.3701287840776049,
      "grad_norm": 3.0003703037058036e-10,
      "learning_rate": 6.598899386669395e-07,
      "logits/chosen": 8.911181449890137,
      "logits/rejected": 8.530158042907715,
      "logps/chosen": -41.685951232910156,
      "logps/rejected": -42.94929504394531,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.685951232910156,
      "rewards/margins": 1.2633416652679443,
      "rewards/rejected": -42.94929504394531,
      "step": 2560
    },
    {
      "epoch": 1.3728048168590066,
      "grad_norm": 1.957785349843777e-12,
      "learning_rate": 6.584135390754679e-07,
      "logits/chosen": 8.727920532226562,
      "logits/rejected": 8.331493377685547,
      "logps/chosen": -41.623451232910156,
      "logps/rejected": -43.342689514160156,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.623451232910156,
      "rewards/margins": 1.719238042831421,
      "rewards/rejected": -43.342689514160156,
      "step": 2565
    },
    {
      "epoch": 1.375480849640408,
      "grad_norm": 4.4989211397416e-08,
      "learning_rate": 6.569356025551454e-07,
      "logits/chosen": 8.65626335144043,
      "logits/rejected": 8.505366325378418,
      "logps/chosen": -42.0156364440918,
      "logps/rejected": -42.59636306762695,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.0156364440918,
      "rewards/margins": 0.5807242393493652,
      "rewards/rejected": -42.59636306762695,
      "step": 2570
    },
    {
      "epoch": 1.3781568824218096,
      "grad_norm": 2.629328258757614e-08,
      "learning_rate": 6.554561434449186e-07,
      "logits/chosen": 8.792577743530273,
      "logits/rejected": 8.441924095153809,
      "logps/chosen": -41.50666809082031,
      "logps/rejected": -42.96664047241211,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.50666809082031,
      "rewards/margins": 1.4599723815917969,
      "rewards/rejected": -42.96664047241211,
      "step": 2575
    },
    {
      "epoch": 1.3808329152032113,
      "grad_norm": 1.8217557630437925e-07,
      "learning_rate": 6.539751760985063e-07,
      "logits/chosen": 8.742188453674316,
      "logits/rejected": 8.44264030456543,
      "logps/chosen": -40.404754638671875,
      "logps/rejected": -41.509552001953125,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -40.404754638671875,
      "rewards/margins": 1.1047991514205933,
      "rewards/rejected": -41.509552001953125,
      "step": 2580
    },
    {
      "epoch": 1.3835089479846128,
      "grad_norm": 8.694552809611475e-08,
      "learning_rate": 6.524927148842602e-07,
      "logits/chosen": 8.650606155395508,
      "logits/rejected": 8.200862884521484,
      "logps/chosen": -42.68472671508789,
      "logps/rejected": -44.271270751953125,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.68472671508789,
      "rewards/margins": 1.5865452289581299,
      "rewards/rejected": -44.271270751953125,
      "step": 2585
    },
    {
      "epoch": 1.3861849807660143,
      "grad_norm": 6.3547481321882486e-15,
      "learning_rate": 6.510087741850254e-07,
      "logits/chosen": 9.042112350463867,
      "logits/rejected": 8.794302940368652,
      "logps/chosen": -42.84346389770508,
      "logps/rejected": -44.69168472290039,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.84346389770508,
      "rewards/margins": 1.8482208251953125,
      "rewards/rejected": -44.69168472290039,
      "step": 2590
    },
    {
      "epoch": 1.388861013547416,
      "grad_norm": 4.923662942049298e-08,
      "learning_rate": 6.495233683980012e-07,
      "logits/chosen": 8.974845886230469,
      "logits/rejected": 8.706317901611328,
      "logps/chosen": -41.35472106933594,
      "logps/rejected": -42.301124572753906,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -41.35472106933594,
      "rewards/margins": 0.9464014172554016,
      "rewards/rejected": -42.301124572753906,
      "step": 2595
    },
    {
      "epoch": 1.3915370463288175,
      "grad_norm": 1.7267148692331677e-05,
      "learning_rate": 6.480365119346011e-07,
      "logits/chosen": 8.6168851852417,
      "logits/rejected": 8.161127090454102,
      "logps/chosen": -40.83750915527344,
      "logps/rejected": -41.776756286621094,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -40.83750915527344,
      "rewards/margins": 0.9392474889755249,
      "rewards/rejected": -41.776756286621094,
      "step": 2600
    },
    {
      "epoch": 1.394213079110219,
      "grad_norm": 6.897164505210194e-07,
      "learning_rate": 6.465482192203129e-07,
      "logits/chosen": 8.35374641418457,
      "logits/rejected": 8.269282341003418,
      "logps/chosen": -41.55049514770508,
      "logps/rejected": -42.83757400512695,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.55049514770508,
      "rewards/margins": 1.2870848178863525,
      "rewards/rejected": -42.83757400512695,
      "step": 2605
    },
    {
      "epoch": 1.3968891118916207,
      "grad_norm": 1.750062726316517e-06,
      "learning_rate": 6.45058504694559e-07,
      "logits/chosen": 8.309649467468262,
      "logits/rejected": 8.38338565826416,
      "logps/chosen": -41.148685455322266,
      "logps/rejected": -43.27328872680664,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -41.148685455322266,
      "rewards/margins": 2.124605178833008,
      "rewards/rejected": -43.27328872680664,
      "step": 2610
    },
    {
      "epoch": 1.3995651446730222,
      "grad_norm": 1.8949276543353387e-05,
      "learning_rate": 6.435673828105564e-07,
      "logits/chosen": 8.728071212768555,
      "logits/rejected": 8.415145874023438,
      "logps/chosen": -39.964942932128906,
      "logps/rejected": -41.9430046081543,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -39.964942932128906,
      "rewards/margins": 1.9780590534210205,
      "rewards/rejected": -41.9430046081543,
      "step": 2615
    },
    {
      "epoch": 1.402241177454424,
      "grad_norm": 2.354581939964938e-12,
      "learning_rate": 6.420748680351763e-07,
      "logits/chosen": 8.721088409423828,
      "logits/rejected": 8.743288040161133,
      "logps/chosen": -41.229286193847656,
      "logps/rejected": -42.19605255126953,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.229286193847656,
      "rewards/margins": 0.9667743444442749,
      "rewards/rejected": -42.19605255126953,
      "step": 2620
    },
    {
      "epoch": 1.4049172102358254,
      "grad_norm": 5.081077099835597e-09,
      "learning_rate": 6.405809748488032e-07,
      "logits/chosen": 8.89842414855957,
      "logits/rejected": 8.644752502441406,
      "logps/chosen": -42.63088607788086,
      "logps/rejected": -43.74695587158203,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.63088607788086,
      "rewards/margins": 1.116062045097351,
      "rewards/rejected": -43.74695587158203,
      "step": 2625
    },
    {
      "epoch": 1.4075932430172269,
      "grad_norm": 7.119297127373652e-11,
      "learning_rate": 6.390857177451956e-07,
      "logits/chosen": 8.8165922164917,
      "logits/rejected": 8.361710548400879,
      "logps/chosen": -41.20853805541992,
      "logps/rejected": -42.973976135253906,
      "loss": 0.0008,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -41.20853805541992,
      "rewards/margins": 1.7654361724853516,
      "rewards/rejected": -42.973976135253906,
      "step": 2630
    },
    {
      "epoch": 1.4102692757986286,
      "grad_norm": 7.145017418345644e-07,
      "learning_rate": 6.375891112313445e-07,
      "logits/chosen": 8.590654373168945,
      "logits/rejected": 8.280073165893555,
      "logps/chosen": -41.35600662231445,
      "logps/rejected": -43.14480972290039,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.35600662231445,
      "rewards/margins": 1.7888002395629883,
      "rewards/rejected": -43.14480972290039,
      "step": 2635
    },
    {
      "epoch": 1.41294530858003,
      "grad_norm": 1.339794923848692e-08,
      "learning_rate": 6.360911698273326e-07,
      "logits/chosen": 8.734606742858887,
      "logits/rejected": 8.351479530334473,
      "logps/chosen": -41.319435119628906,
      "logps/rejected": -42.49150085449219,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -41.319435119628906,
      "rewards/margins": 1.172070860862732,
      "rewards/rejected": -42.49150085449219,
      "step": 2640
    },
    {
      "epoch": 1.4156213413614318,
      "grad_norm": 3.9412070211162934e-11,
      "learning_rate": 6.345919080661944e-07,
      "logits/chosen": 8.945253372192383,
      "logits/rejected": 8.674893379211426,
      "logps/chosen": -42.319496154785156,
      "logps/rejected": -43.33086395263672,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.319496154785156,
      "rewards/margins": 1.0113706588745117,
      "rewards/rejected": -43.33086395263672,
      "step": 2645
    },
    {
      "epoch": 1.4182973741428333,
      "grad_norm": 4.162284700657952e-10,
      "learning_rate": 6.330913404937737e-07,
      "logits/chosen": 8.95710563659668,
      "logits/rejected": 8.489824295043945,
      "logps/chosen": -42.271392822265625,
      "logps/rejected": -43.92520523071289,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -42.271392822265625,
      "rewards/margins": 1.6538140773773193,
      "rewards/rejected": -43.92520523071289,
      "step": 2650
    },
    {
      "epoch": 1.4209734069242348,
      "grad_norm": 2.5087827651696878e-05,
      "learning_rate": 6.315894816685838e-07,
      "logits/chosen": 8.598902702331543,
      "logits/rejected": 8.227259635925293,
      "logps/chosen": -42.930458068847656,
      "logps/rejected": -43.603599548339844,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.930458068847656,
      "rewards/margins": 0.6731430888175964,
      "rewards/rejected": -43.603599548339844,
      "step": 2655
    },
    {
      "epoch": 1.4236494397056365,
      "grad_norm": 5.030919522031955e-13,
      "learning_rate": 6.300863461616657e-07,
      "logits/chosen": 8.209463119506836,
      "logits/rejected": 7.915236473083496,
      "logps/chosen": -42.28984832763672,
      "logps/rejected": -43.53640365600586,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -42.28984832763672,
      "rewards/margins": 1.2465569972991943,
      "rewards/rejected": -43.53640365600586,
      "step": 2660
    },
    {
      "epoch": 1.426325472487038,
      "grad_norm": 2.8482206258726657e-09,
      "learning_rate": 6.285819485564465e-07,
      "logits/chosen": 9.046075820922852,
      "logits/rejected": 8.36937141418457,
      "logps/chosen": -41.806270599365234,
      "logps/rejected": -42.67213439941406,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -41.806270599365234,
      "rewards/margins": 0.8658647537231445,
      "rewards/rejected": -42.67213439941406,
      "step": 2665
    },
    {
      "epoch": 1.4290015052684395,
      "grad_norm": 3.7938968593213507e-07,
      "learning_rate": 6.270763034485986e-07,
      "logits/chosen": 8.585349082946777,
      "logits/rejected": 8.288152694702148,
      "logps/chosen": -40.58457946777344,
      "logps/rejected": -41.821144104003906,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.58457946777344,
      "rewards/margins": 1.236563801765442,
      "rewards/rejected": -41.821144104003906,
      "step": 2670
    },
    {
      "epoch": 1.4316775380498412,
      "grad_norm": 8.576400073891725e-06,
      "learning_rate": 6.255694254458972e-07,
      "logits/chosen": 8.569849967956543,
      "logits/rejected": 8.177288055419922,
      "logps/chosen": -40.41807174682617,
      "logps/rejected": -42.742557525634766,
      "loss": 0.0033,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -40.41807174682617,
      "rewards/margins": 2.3244881629943848,
      "rewards/rejected": -42.742557525634766,
      "step": 2675
    },
    {
      "epoch": 1.4343535708312427,
      "grad_norm": 1.5803549186674131e-06,
      "learning_rate": 6.240613291680795e-07,
      "logits/chosen": 8.672123908996582,
      "logits/rejected": 8.24134349822998,
      "logps/chosen": -42.231529235839844,
      "logps/rejected": -44.24704360961914,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -42.231529235839844,
      "rewards/margins": 2.0155093669891357,
      "rewards/rejected": -44.24704360961914,
      "step": 2680
    },
    {
      "epoch": 1.4370296036126442,
      "grad_norm": 0.0003616355373746601,
      "learning_rate": 6.225520292467021e-07,
      "logits/chosen": 8.974054336547852,
      "logits/rejected": 8.090018272399902,
      "logps/chosen": -41.403297424316406,
      "logps/rejected": -43.79715347290039,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.403297424316406,
      "rewards/margins": 2.3938541412353516,
      "rewards/rejected": -43.79715347290039,
      "step": 2685
    },
    {
      "epoch": 1.439705636394046,
      "grad_norm": 3.955351860021308e-11,
      "learning_rate": 6.210415403249993e-07,
      "logits/chosen": 8.829411506652832,
      "logits/rejected": 8.101358413696289,
      "logps/chosen": -40.76591491699219,
      "logps/rejected": -43.67510223388672,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -40.76591491699219,
      "rewards/margins": 2.909186363220215,
      "rewards/rejected": -43.67510223388672,
      "step": 2690
    },
    {
      "epoch": 1.4423816691754474,
      "grad_norm": 1.2346477441096996e-06,
      "learning_rate": 6.195298770577415e-07,
      "logits/chosen": 8.410456657409668,
      "logits/rejected": 8.339765548706055,
      "logps/chosen": -42.111656188964844,
      "logps/rejected": -42.86857986450195,
      "loss": 0.0002,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -42.111656188964844,
      "rewards/margins": 0.7569217681884766,
      "rewards/rejected": -42.86857986450195,
      "step": 2695
    },
    {
      "epoch": 1.445057701956849,
      "grad_norm": 1.9201134335457046e-07,
      "learning_rate": 6.180170541110923e-07,
      "logits/chosen": 8.710095405578613,
      "logits/rejected": 8.143321990966797,
      "logps/chosen": -41.566593170166016,
      "logps/rejected": -42.4012336730957,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -41.566593170166016,
      "rewards/margins": 0.8346397280693054,
      "rewards/rejected": -42.4012336730957,
      "step": 2700
    },
    {
      "epoch": 1.4477337347382506,
      "grad_norm": 6.147084902213062e-09,
      "learning_rate": 6.165030861624663e-07,
      "logits/chosen": 9.157144546508789,
      "logits/rejected": 8.170949935913086,
      "logps/chosen": -41.26401901245117,
      "logps/rejected": -42.615875244140625,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.26401901245117,
      "rewards/margins": 1.3518584966659546,
      "rewards/rejected": -42.615875244140625,
      "step": 2705
    },
    {
      "epoch": 1.4504097675196521,
      "grad_norm": 1.4680937589798419e-08,
      "learning_rate": 6.149879879003876e-07,
      "logits/chosen": 8.618410110473633,
      "logits/rejected": 8.747294425964355,
      "logps/chosen": -41.62251663208008,
      "logps/rejected": -42.37772750854492,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.62251663208008,
      "rewards/margins": 0.7552111744880676,
      "rewards/rejected": -42.37772750854492,
      "step": 2710
    },
    {
      "epoch": 1.4530858003010536,
      "grad_norm": 5.780344334401673e-07,
      "learning_rate": 6.13471774024346e-07,
      "logits/chosen": 8.884693145751953,
      "logits/rejected": 8.513826370239258,
      "logps/chosen": -40.83106231689453,
      "logps/rejected": -41.84722137451172,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -40.83106231689453,
      "rewards/margins": 1.0161610841751099,
      "rewards/rejected": -41.84722137451172,
      "step": 2715
    },
    {
      "epoch": 1.4557618330824553,
      "grad_norm": 3.316326782492241e-09,
      "learning_rate": 6.119544592446551e-07,
      "logits/chosen": 8.547552108764648,
      "logits/rejected": 8.374757766723633,
      "logps/chosen": -41.661956787109375,
      "logps/rejected": -43.59321975708008,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.661956787109375,
      "rewards/margins": 1.931265115737915,
      "rewards/rejected": -43.59321975708008,
      "step": 2720
    },
    {
      "epoch": 1.4584378658638568,
      "grad_norm": 1.304987298256512e-07,
      "learning_rate": 6.104360582823096e-07,
      "logits/chosen": 8.907578468322754,
      "logits/rejected": 8.626019477844238,
      "logps/chosen": -40.874298095703125,
      "logps/rejected": -43.00669479370117,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -40.874298095703125,
      "rewards/margins": 2.132399320602417,
      "rewards/rejected": -43.00669479370117,
      "step": 2725
    },
    {
      "epoch": 1.4611138986452583,
      "grad_norm": 2.5052392047264423e-09,
      "learning_rate": 6.089165858688423e-07,
      "logits/chosen": 8.702375411987305,
      "logits/rejected": 8.062834739685059,
      "logps/chosen": -41.73476028442383,
      "logps/rejected": -42.20301818847656,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.73476028442383,
      "rewards/margins": 0.4682607054710388,
      "rewards/rejected": -42.20301818847656,
      "step": 2730
    },
    {
      "epoch": 1.46378993142666,
      "grad_norm": 1.4971702048636752e-07,
      "learning_rate": 6.073960567461811e-07,
      "logits/chosen": 9.04012680053711,
      "logits/rejected": 8.473031997680664,
      "logps/chosen": -41.67736053466797,
      "logps/rejected": -43.5404052734375,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -41.67736053466797,
      "rewards/margins": 1.8630421161651611,
      "rewards/rejected": -43.5404052734375,
      "step": 2735
    },
    {
      "epoch": 1.4664659642080615,
      "grad_norm": 8.926331926282415e-09,
      "learning_rate": 6.058744856665065e-07,
      "logits/chosen": 8.426590919494629,
      "logits/rejected": 8.039485931396484,
      "logps/chosen": -41.57054138183594,
      "logps/rejected": -43.211910247802734,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -41.57054138183594,
      "rewards/margins": 1.6413624286651611,
      "rewards/rejected": -43.211910247802734,
      "step": 2740
    },
    {
      "epoch": 1.469141996989463,
      "grad_norm": 8.14014674716484e-05,
      "learning_rate": 6.043518873921074e-07,
      "logits/chosen": 9.057416915893555,
      "logits/rejected": 8.462617874145508,
      "logps/chosen": -40.877845764160156,
      "logps/rejected": -42.28983688354492,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -40.877845764160156,
      "rewards/margins": 1.4119914770126343,
      "rewards/rejected": -42.28983688354492,
      "step": 2745
    },
    {
      "epoch": 1.4718180297708647,
      "grad_norm": 5.254469351115687e-09,
      "learning_rate": 6.028282766952393e-07,
      "logits/chosen": 8.44697380065918,
      "logits/rejected": 8.054309844970703,
      "logps/chosen": -41.75263214111328,
      "logps/rejected": -42.920040130615234,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -41.75263214111328,
      "rewards/margins": 1.1674166917800903,
      "rewards/rejected": -42.920040130615234,
      "step": 2750
    },
    {
      "epoch": 1.4744940625522662,
      "grad_norm": 8.455064514128113e-09,
      "learning_rate": 6.013036683579798e-07,
      "logits/chosen": 8.765707015991211,
      "logits/rejected": 8.471872329711914,
      "logps/chosen": -41.94719314575195,
      "logps/rejected": -43.32516098022461,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -41.94719314575195,
      "rewards/margins": 1.3779693841934204,
      "rewards/rejected": -43.32516098022461,
      "step": 2755
    },
    {
      "epoch": 1.4771700953336677,
      "grad_norm": 5.591991893940422e-05,
      "learning_rate": 5.997780771720854e-07,
      "logits/chosen": 8.819055557250977,
      "logits/rejected": 8.026742935180664,
      "logps/chosen": -40.92245101928711,
      "logps/rejected": -42.21430587768555,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -40.92245101928711,
      "rewards/margins": 1.2918500900268555,
      "rewards/rejected": -42.21430587768555,
      "step": 2760
    },
    {
      "epoch": 1.4798461281150694,
      "grad_norm": 2.967908968636662e-14,
      "learning_rate": 5.982515179388486e-07,
      "logits/chosen": 8.710647583007812,
      "logits/rejected": 8.366626739501953,
      "logps/chosen": -42.1692008972168,
      "logps/rejected": -43.35497283935547,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -42.1692008972168,
      "rewards/margins": 1.1857683658599854,
      "rewards/rejected": -43.35497283935547,
      "step": 2765
    },
    {
      "epoch": 1.482522160896471,
      "grad_norm": 0.00014213200505656836,
      "learning_rate": 5.967240054689541e-07,
      "logits/chosen": 9.115320205688477,
      "logits/rejected": 9.052719116210938,
      "logps/chosen": -41.38245391845703,
      "logps/rejected": -42.613563537597656,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -41.38245391845703,
      "rewards/margins": 1.2311127185821533,
      "rewards/rejected": -42.613563537597656,
      "step": 2770
    },
    {
      "epoch": 1.4851981936778724,
      "grad_norm": 1.2366252018341718e-06,
      "learning_rate": 5.951955545823342e-07,
      "logits/chosen": 8.947187423706055,
      "logits/rejected": 8.704947471618652,
      "logps/chosen": -41.465145111083984,
      "logps/rejected": -43.29473876953125,
      "loss": 0.0,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -41.465145111083984,
      "rewards/margins": 1.829595923423767,
      "rewards/rejected": -43.29473876953125,
      "step": 2775
    },
    {
      "epoch": 1.4878742264592741,
      "grad_norm": 6.5358773561707e-09,
      "learning_rate": 5.936661801080263e-07,
      "logits/chosen": 8.623039245605469,
      "logits/rejected": 8.29152774810791,
      "logps/chosen": -41.36957550048828,
      "logps/rejected": -42.660457611083984,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -41.36957550048828,
      "rewards/margins": 1.2908824682235718,
      "rewards/rejected": -42.660457611083984,
      "step": 2780
    },
    {
      "epoch": 1.4905502592406756,
      "grad_norm": 1.4478548016769985e-07,
      "learning_rate": 5.92135896884028e-07,
      "logits/chosen": 9.315587997436523,
      "logits/rejected": 8.763214111328125,
      "logps/chosen": -42.89904022216797,
      "logps/rejected": -43.45850372314453,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.89904022216797,
      "rewards/margins": 0.559464156627655,
      "rewards/rejected": -43.45850372314453,
      "step": 2785
    },
    {
      "epoch": 1.4932262920220774,
      "grad_norm": 4.6752017095706123e-07,
      "learning_rate": 5.906047197571541e-07,
      "logits/chosen": 8.681356430053711,
      "logits/rejected": 8.956047058105469,
      "logps/chosen": -42.605445861816406,
      "logps/rejected": -42.976097106933594,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.605445861816406,
      "rewards/margins": 0.37064868211746216,
      "rewards/rejected": -42.976097106933594,
      "step": 2790
    },
    {
      "epoch": 1.4959023248034788,
      "grad_norm": 1.3092466133363111e-06,
      "learning_rate": 5.890726635828919e-07,
      "logits/chosen": 8.810511589050293,
      "logits/rejected": 8.950778007507324,
      "logps/chosen": -42.79461669921875,
      "logps/rejected": -43.614627838134766,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -42.79461669921875,
      "rewards/margins": 0.8200146555900574,
      "rewards/rejected": -43.614627838134766,
      "step": 2795
    },
    {
      "epoch": 1.4985783575848803,
      "grad_norm": 1.1611370573916827e-09,
      "learning_rate": 5.875397432252569e-07,
      "logits/chosen": 8.87797737121582,
      "logits/rejected": 8.874384880065918,
      "logps/chosen": -41.66436004638672,
      "logps/rejected": -41.99811553955078,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -41.66436004638672,
      "rewards/margins": 0.3337518572807312,
      "rewards/rejected": -41.99811553955078,
      "step": 2800
    },
    {
      "epoch": 1.4985783575848803,
      "eval_logits/chosen": 7.462876796722412,
      "eval_logits/rejected": 7.207109451293945,
      "eval_logps/chosen": -41.718467712402344,
      "eval_logps/rejected": -42.90156555175781,
      "eval_loss": 2.9896133924012247e-07,
      "eval_rewards/accuracies": 0.5630564093589783,
      "eval_rewards/chosen": -41.718467712402344,
      "eval_rewards/margins": 1.1831080913543701,
      "eval_rewards/rejected": -42.90156555175781,
      "eval_runtime": 40.3486,
      "eval_samples_per_second": 33.334,
      "eval_steps_per_second": 8.352,
      "step": 2800
    },
    {
      "epoch": 1.5012543903662818,
      "grad_norm": 5.240221569606272e-09,
      "learning_rate": 5.860059735566491e-07,
      "logits/chosen": 9.308040618896484,
      "logits/rejected": 8.846551895141602,
      "logps/chosen": -42.58149337768555,
      "logps/rejected": -43.26437759399414,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.58149337768555,
      "rewards/margins": 0.6828859448432922,
      "rewards/rejected": -43.26437759399414,
      "step": 2805
    },
    {
      "epoch": 1.5039304231476835,
      "grad_norm": 1.5241795381715775e-10,
      "learning_rate": 5.844713694577087e-07,
      "logits/chosen": 8.802517890930176,
      "logits/rejected": 8.490335464477539,
      "logps/chosen": -41.10953903198242,
      "logps/rejected": -42.98762893676758,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -41.10953903198242,
      "rewards/margins": 1.8780874013900757,
      "rewards/rejected": -42.98762893676758,
      "step": 2810
    },
    {
      "epoch": 1.5066064559290853,
      "grad_norm": 0.00014955291429823763,
      "learning_rate": 5.829359458171714e-07,
      "logits/chosen": 8.675622940063477,
      "logits/rejected": 8.524667739868164,
      "logps/chosen": -42.0042724609375,
      "logps/rejected": -43.8996696472168,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.0042724609375,
      "rewards/margins": 1.895399808883667,
      "rewards/rejected": -43.8996696472168,
      "step": 2815
    },
    {
      "epoch": 1.5092824887104868,
      "grad_norm": 4.446093561651825e-05,
      "learning_rate": 5.81399717531724e-07,
      "logits/chosen": 9.089492797851562,
      "logits/rejected": 8.260062217712402,
      "logps/chosen": -41.917442321777344,
      "logps/rejected": -43.20248794555664,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -41.917442321777344,
      "rewards/margins": 1.285048246383667,
      "rewards/rejected": -43.20248794555664,
      "step": 2820
    },
    {
      "epoch": 1.5119585214918883,
      "grad_norm": 5.8508346770030255e-05,
      "learning_rate": 5.798626995058602e-07,
      "logits/chosen": 9.176536560058594,
      "logits/rejected": 8.630223274230957,
      "logps/chosen": -41.753700256347656,
      "logps/rejected": -42.61920166015625,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -41.753700256347656,
      "rewards/margins": 0.8655007481575012,
      "rewards/rejected": -42.61920166015625,
      "step": 2825
    },
    {
      "epoch": 1.51463455427329,
      "grad_norm": 6.5935377030015784e-12,
      "learning_rate": 5.783249066517354e-07,
      "logits/chosen": 8.406403541564941,
      "logits/rejected": 8.233708381652832,
      "logps/chosen": -42.976043701171875,
      "logps/rejected": -44.048439025878906,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.976043701171875,
      "rewards/margins": 1.0723955631256104,
      "rewards/rejected": -44.048439025878906,
      "step": 2830
    },
    {
      "epoch": 1.5173105870546915,
      "grad_norm": 3.629062625186188e-07,
      "learning_rate": 5.767863538890228e-07,
      "logits/chosen": 8.886978149414062,
      "logits/rejected": 8.5150146484375,
      "logps/chosen": -41.98692321777344,
      "logps/rejected": -43.80683517456055,
      "loss": 0.0006,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -41.98692321777344,
      "rewards/margins": 1.8199154138565063,
      "rewards/rejected": -43.80683517456055,
      "step": 2835
    },
    {
      "epoch": 1.519986619836093,
      "grad_norm": 2.3701190253475095e-09,
      "learning_rate": 5.75247056144768e-07,
      "logits/chosen": 8.84416389465332,
      "logits/rejected": 8.61463737487793,
      "logps/chosen": -42.14787673950195,
      "logps/rejected": -43.078182220458984,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.14787673950195,
      "rewards/margins": 0.9303110241889954,
      "rewards/rejected": -43.078182220458984,
      "step": 2840
    },
    {
      "epoch": 1.5226626526174947,
      "grad_norm": 5.131912228535976e-08,
      "learning_rate": 5.737070283532444e-07,
      "logits/chosen": 9.2601900100708,
      "logits/rejected": 8.791196823120117,
      "logps/chosen": -42.051273345947266,
      "logps/rejected": -43.37104797363281,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -42.051273345947266,
      "rewards/margins": 1.319777250289917,
      "rewards/rejected": -43.37104797363281,
      "step": 2845
    },
    {
      "epoch": 1.5253386853988962,
      "grad_norm": 2.412108424898287e-11,
      "learning_rate": 5.721662854558084e-07,
      "logits/chosen": 8.788159370422363,
      "logits/rejected": 8.487028121948242,
      "logps/chosen": -41.32537841796875,
      "logps/rejected": -42.331199645996094,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -41.32537841796875,
      "rewards/margins": 1.005817174911499,
      "rewards/rejected": -42.331199645996094,
      "step": 2850
    },
    {
      "epoch": 1.5280147181802977,
      "grad_norm": 4.261682624227416e-12,
      "learning_rate": 5.706248424007545e-07,
      "logits/chosen": 8.630437850952148,
      "logits/rejected": 8.20901870727539,
      "logps/chosen": -40.639930725097656,
      "logps/rejected": -42.065704345703125,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -40.639930725097656,
      "rewards/margins": 1.4257698059082031,
      "rewards/rejected": -42.065704345703125,
      "step": 2855
    },
    {
      "epoch": 1.5306907509616994,
      "grad_norm": 7.364941016173929e-05,
      "learning_rate": 5.690827141431699e-07,
      "logits/chosen": 9.075970649719238,
      "logits/rejected": 8.477521896362305,
      "logps/chosen": -42.701927185058594,
      "logps/rejected": -43.13222885131836,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.701927185058594,
      "rewards/margins": 0.43030399084091187,
      "rewards/rejected": -43.13222885131836,
      "step": 2860
    },
    {
      "epoch": 1.5333667837431009,
      "grad_norm": 2.2940298544455694e-10,
      "learning_rate": 5.675399156447897e-07,
      "logits/chosen": 9.055513381958008,
      "logits/rejected": 8.465234756469727,
      "logps/chosen": -41.51704025268555,
      "logps/rejected": -42.361717224121094,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.51704025268555,
      "rewards/margins": 0.8446768522262573,
      "rewards/rejected": -42.361717224121094,
      "step": 2865
    },
    {
      "epoch": 1.5360428165245024,
      "grad_norm": 0.0004173381613143238,
      "learning_rate": 5.659964618738515e-07,
      "logits/chosen": 9.119138717651367,
      "logits/rejected": 8.610302925109863,
      "logps/chosen": -41.553932189941406,
      "logps/rejected": -42.481388092041016,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -41.553932189941406,
      "rewards/margins": 0.9274513125419617,
      "rewards/rejected": -42.481388092041016,
      "step": 2870
    },
    {
      "epoch": 1.538718849305904,
      "grad_norm": 5.233463765742127e-05,
      "learning_rate": 5.644523678049509e-07,
      "logits/chosen": 8.967458724975586,
      "logits/rejected": 8.663304328918457,
      "logps/chosen": -41.5849723815918,
      "logps/rejected": -42.68383026123047,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -41.5849723815918,
      "rewards/margins": 1.0988551378250122,
      "rewards/rejected": -42.68383026123047,
      "step": 2875
    },
    {
      "epoch": 1.5413948820873056,
      "grad_norm": 2.9087525451153584e-05,
      "learning_rate": 5.629076484188952e-07,
      "logits/chosen": 8.666072845458984,
      "logits/rejected": 8.390230178833008,
      "logps/chosen": -42.19350814819336,
      "logps/rejected": -43.1552734375,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.19350814819336,
      "rewards/margins": 0.9617654085159302,
      "rewards/rejected": -43.1552734375,
      "step": 2880
    },
    {
      "epoch": 1.544070914868707,
      "grad_norm": 2.9015629756792404e-09,
      "learning_rate": 5.613623187025587e-07,
      "logits/chosen": 9.060524940490723,
      "logits/rejected": 8.695516586303711,
      "logps/chosen": -43.05659103393555,
      "logps/rejected": -43.17965316772461,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.05659103393555,
      "rewards/margins": 0.12306338548660278,
      "rewards/rejected": -43.17965316772461,
      "step": 2885
    },
    {
      "epoch": 1.5467469476501088,
      "grad_norm": 2.4957535969279874e-08,
      "learning_rate": 5.598163936487369e-07,
      "logits/chosen": 8.534125328063965,
      "logits/rejected": 7.987432956695557,
      "logps/chosen": -39.602210998535156,
      "logps/rejected": -41.279685974121094,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -39.602210998535156,
      "rewards/margins": 1.677476167678833,
      "rewards/rejected": -41.279685974121094,
      "step": 2890
    },
    {
      "epoch": 1.5494229804315103,
      "grad_norm": 2.8436341546900504e-07,
      "learning_rate": 5.582698882560017e-07,
      "logits/chosen": 9.013283729553223,
      "logits/rejected": 8.313443183898926,
      "logps/chosen": -42.68204116821289,
      "logps/rejected": -43.479148864746094,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.68204116821289,
      "rewards/margins": 0.7971112728118896,
      "rewards/rejected": -43.479148864746094,
      "step": 2895
    },
    {
      "epoch": 1.5520990132129118,
      "grad_norm": 2.079327247052572e-08,
      "learning_rate": 5.567228175285549e-07,
      "logits/chosen": 8.712634086608887,
      "logits/rejected": 8.37256908416748,
      "logps/chosen": -42.44874572753906,
      "logps/rejected": -42.86425018310547,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -42.44874572753906,
      "rewards/margins": 0.41550564765930176,
      "rewards/rejected": -42.86425018310547,
      "step": 2900
    },
    {
      "epoch": 1.5547750459943135,
      "grad_norm": 5.73634186703197e-10,
      "learning_rate": 5.551751964760838e-07,
      "logits/chosen": 8.806078910827637,
      "logits/rejected": 8.834476470947266,
      "logps/chosen": -42.70290756225586,
      "logps/rejected": -43.19768524169922,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.70290756225586,
      "rewards/margins": 0.4947790503501892,
      "rewards/rejected": -43.19768524169922,
      "step": 2905
    },
    {
      "epoch": 1.557451078775715,
      "grad_norm": 2.8021702646588545e-09,
      "learning_rate": 5.536270401136145e-07,
      "logits/chosen": 8.93919849395752,
      "logits/rejected": 8.48941707611084,
      "logps/chosen": -42.132591247558594,
      "logps/rejected": -42.882015228271484,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.132591247558594,
      "rewards/margins": 0.7494180798530579,
      "rewards/rejected": -42.882015228271484,
      "step": 2910
    },
    {
      "epoch": 1.5601271115571165,
      "grad_norm": 4.64860085575252e-08,
      "learning_rate": 5.520783634613667e-07,
      "logits/chosen": 8.664732933044434,
      "logits/rejected": 8.2017240524292,
      "logps/chosen": -42.85877227783203,
      "logps/rejected": -43.6851921081543,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.85877227783203,
      "rewards/margins": 0.8264285922050476,
      "rewards/rejected": -43.6851921081543,
      "step": 2915
    },
    {
      "epoch": 1.5628031443385182,
      "grad_norm": 8.214540923455313e-10,
      "learning_rate": 5.505291815446082e-07,
      "logits/chosen": 8.691978454589844,
      "logits/rejected": 8.322660446166992,
      "logps/chosen": -41.80618667602539,
      "logps/rejected": -43.3055534362793,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.80618667602539,
      "rewards/margins": 1.499367356300354,
      "rewards/rejected": -43.3055534362793,
      "step": 2920
    },
    {
      "epoch": 1.5654791771199197,
      "grad_norm": 1.1086264503927307e-08,
      "learning_rate": 5.489795093935089e-07,
      "logits/chosen": 8.877646446228027,
      "logits/rejected": 8.47821044921875,
      "logps/chosen": -42.01649856567383,
      "logps/rejected": -42.49527359008789,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.01649856567383,
      "rewards/margins": 0.4787684977054596,
      "rewards/rejected": -42.49527359008789,
      "step": 2925
    },
    {
      "epoch": 1.5681552099013212,
      "grad_norm": 1.6309603960220027e-09,
      "learning_rate": 5.474293620429946e-07,
      "logits/chosen": 8.955114364624023,
      "logits/rejected": 8.372546195983887,
      "logps/chosen": -41.6548957824707,
      "logps/rejected": -43.28264617919922,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -41.6548957824707,
      "rewards/margins": 1.6277482509613037,
      "rewards/rejected": -43.28264617919922,
      "step": 2930
    },
    {
      "epoch": 1.570831242682723,
      "grad_norm": 0.00011882941871267415,
      "learning_rate": 5.458787545326018e-07,
      "logits/chosen": 8.98145580291748,
      "logits/rejected": 8.75607681274414,
      "logps/chosen": -41.506309509277344,
      "logps/rejected": -42.45687484741211,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -41.506309509277344,
      "rewards/margins": 0.9505647420883179,
      "rewards/rejected": -42.45687484741211,
      "step": 2935
    },
    {
      "epoch": 1.5735072754641244,
      "grad_norm": 3.113898426379346e-06,
      "learning_rate": 5.443277019063311e-07,
      "logits/chosen": 9.05129337310791,
      "logits/rejected": 8.804384231567383,
      "logps/chosen": -42.770999908447266,
      "logps/rejected": -44.222930908203125,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -42.770999908447266,
      "rewards/margins": 1.4519296884536743,
      "rewards/rejected": -44.222930908203125,
      "step": 2940
    },
    {
      "epoch": 1.5761833082455259,
      "grad_norm": 2.8471975020122704e-07,
      "learning_rate": 5.427762192125023e-07,
      "logits/chosen": 8.79440689086914,
      "logits/rejected": 8.30298900604248,
      "logps/chosen": -42.04811096191406,
      "logps/rejected": -42.66920471191406,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.04811096191406,
      "rewards/margins": 0.6210967302322388,
      "rewards/rejected": -42.66920471191406,
      "step": 2945
    },
    {
      "epoch": 1.5788593410269276,
      "grad_norm": 1.551237060611357e-07,
      "learning_rate": 5.41224321503607e-07,
      "logits/chosen": 8.793144226074219,
      "logits/rejected": 8.105385780334473,
      "logps/chosen": -41.787071228027344,
      "logps/rejected": -44.501731872558594,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -41.787071228027344,
      "rewards/margins": 2.714656114578247,
      "rewards/rejected": -44.501731872558594,
      "step": 2950
    },
    {
      "epoch": 1.5815353738083293,
      "grad_norm": 0.00012709212277160943,
      "learning_rate": 5.396720238361637e-07,
      "logits/chosen": 8.662508964538574,
      "logits/rejected": 8.256942749023438,
      "logps/chosen": -41.13361358642578,
      "logps/rejected": -42.892234802246094,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.13361358642578,
      "rewards/margins": 1.758623480796814,
      "rewards/rejected": -42.892234802246094,
      "step": 2955
    },
    {
      "epoch": 1.5842114065897306,
      "grad_norm": 4.068168534742173e-06,
      "learning_rate": 5.381193412705711e-07,
      "logits/chosen": 8.76036262512207,
      "logits/rejected": 8.307029724121094,
      "logps/chosen": -40.78772735595703,
      "logps/rejected": -41.676429748535156,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.78772735595703,
      "rewards/margins": 0.8887056112289429,
      "rewards/rejected": -41.676429748535156,
      "step": 2960
    },
    {
      "epoch": 1.5868874393711323,
      "grad_norm": 2.706116894915754e-09,
      "learning_rate": 5.365662888709622e-07,
      "logits/chosen": 8.910306930541992,
      "logits/rejected": 8.499940872192383,
      "logps/chosen": -41.561187744140625,
      "logps/rejected": -42.84279251098633,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.561187744140625,
      "rewards/margins": 1.2816061973571777,
      "rewards/rejected": -42.84279251098633,
      "step": 2965
    },
    {
      "epoch": 1.589563472152534,
      "grad_norm": 5.775261032822579e-09,
      "learning_rate": 5.350128817050585e-07,
      "logits/chosen": 9.142675399780273,
      "logits/rejected": 8.506601333618164,
      "logps/chosen": -42.51160430908203,
      "logps/rejected": -43.86318588256836,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.51160430908203,
      "rewards/margins": 1.3515797853469849,
      "rewards/rejected": -43.86318588256836,
      "step": 2970
    },
    {
      "epoch": 1.5922395049339353,
      "grad_norm": 3.6340854899170017e-09,
      "learning_rate": 5.334591348440229e-07,
      "logits/chosen": 8.863957405090332,
      "logits/rejected": 8.378945350646973,
      "logps/chosen": -41.649898529052734,
      "logps/rejected": -42.979164123535156,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -41.649898529052734,
      "rewards/margins": 1.3292641639709473,
      "rewards/rejected": -42.979164123535156,
      "step": 2975
    },
    {
      "epoch": 1.594915537715337,
      "grad_norm": 1.7223451805564094e-09,
      "learning_rate": 5.319050633623141e-07,
      "logits/chosen": 8.876564025878906,
      "logits/rejected": 8.450920104980469,
      "logps/chosen": -41.80146026611328,
      "logps/rejected": -43.48046875,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.80146026611328,
      "rewards/margins": 1.6790072917938232,
      "rewards/rejected": -43.48046875,
      "step": 2980
    },
    {
      "epoch": 1.5975915704967387,
      "grad_norm": 6.255182515691606e-10,
      "learning_rate": 5.303506823375409e-07,
      "logits/chosen": 8.682907104492188,
      "logits/rejected": 8.112383842468262,
      "logps/chosen": -41.28458023071289,
      "logps/rejected": -43.36217498779297,
      "loss": 0.0,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -41.28458023071289,
      "rewards/margins": 2.07759690284729,
      "rewards/rejected": -43.36217498779297,
      "step": 2985
    },
    {
      "epoch": 1.60026760327814,
      "grad_norm": 2.2504293400865813e-11,
      "learning_rate": 5.287960068503143e-07,
      "logits/chosen": 9.328814506530762,
      "logits/rejected": 8.755146980285645,
      "logps/chosen": -42.91911697387695,
      "logps/rejected": -44.47560501098633,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.91911697387695,
      "rewards/margins": 1.5564887523651123,
      "rewards/rejected": -44.47560501098633,
      "step": 2990
    },
    {
      "epoch": 1.6029436360595417,
      "grad_norm": 3.319810814356769e-10,
      "learning_rate": 5.272410519841032e-07,
      "logits/chosen": 8.68800163269043,
      "logits/rejected": 8.423990249633789,
      "logps/chosen": -41.551368713378906,
      "logps/rejected": -42.98131561279297,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.551368713378906,
      "rewards/margins": 1.4299461841583252,
      "rewards/rejected": -42.98131561279297,
      "step": 2995
    },
    {
      "epoch": 1.6056196688409434,
      "grad_norm": 0.00010166629368870996,
      "learning_rate": 5.256858328250861e-07,
      "logits/chosen": 9.132532119750977,
      "logits/rejected": 8.534920692443848,
      "logps/chosen": -42.26913070678711,
      "logps/rejected": -43.59767150878906,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.26913070678711,
      "rewards/margins": 1.328540563583374,
      "rewards/rejected": -43.59767150878906,
      "step": 3000
    },
    {
      "epoch": 1.608295701622345,
      "grad_norm": 3.233041001418665e-07,
      "learning_rate": 5.241303644620063e-07,
      "logits/chosen": 9.308691024780273,
      "logits/rejected": 8.77791976928711,
      "logps/chosen": -43.174034118652344,
      "logps/rejected": -43.86066818237305,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -43.174034118652344,
      "rewards/margins": 0.6866382360458374,
      "rewards/rejected": -43.86066818237305,
      "step": 3005
    },
    {
      "epoch": 1.6109717344037464,
      "grad_norm": 2.3349442928205522e-08,
      "learning_rate": 5.225746619860248e-07,
      "logits/chosen": 9.183420181274414,
      "logits/rejected": 8.742216110229492,
      "logps/chosen": -42.111778259277344,
      "logps/rejected": -43.35272979736328,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.111778259277344,
      "rewards/margins": 1.2409462928771973,
      "rewards/rejected": -43.35272979736328,
      "step": 3010
    },
    {
      "epoch": 1.6136477671851481,
      "grad_norm": 3.0978312359132904e-05,
      "learning_rate": 5.210187404905735e-07,
      "logits/chosen": 8.602811813354492,
      "logits/rejected": 8.375425338745117,
      "logps/chosen": -43.508853912353516,
      "logps/rejected": -43.21955108642578,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.508853912353516,
      "rewards/margins": -0.28930503129959106,
      "rewards/rejected": -43.21955108642578,
      "step": 3015
    },
    {
      "epoch": 1.6163237999665496,
      "grad_norm": 8.028707284773641e-07,
      "learning_rate": 5.194626150712098e-07,
      "logits/chosen": 9.231069564819336,
      "logits/rejected": 8.754720687866211,
      "logps/chosen": -42.7666015625,
      "logps/rejected": -43.946319580078125,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.7666015625,
      "rewards/margins": 1.179720401763916,
      "rewards/rejected": -43.946319580078125,
      "step": 3020
    },
    {
      "epoch": 1.6189998327479511,
      "grad_norm": 1.2500797381750772e-05,
      "learning_rate": 5.179063008254695e-07,
      "logits/chosen": 8.906530380249023,
      "logits/rejected": 8.253819465637207,
      "logps/chosen": -41.62322998046875,
      "logps/rejected": -42.454429626464844,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.62322998046875,
      "rewards/margins": 0.8311986923217773,
      "rewards/rejected": -42.454429626464844,
      "step": 3025
    },
    {
      "epoch": 1.6216758655293528,
      "grad_norm": 0.2365918569251992,
      "learning_rate": 5.163498128527199e-07,
      "logits/chosen": 8.86352825164795,
      "logits/rejected": 8.38776969909668,
      "logps/chosen": -41.10457992553711,
      "logps/rejected": -43.19374084472656,
      "loss": 0.0004,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.10457992553711,
      "rewards/margins": 2.089158296585083,
      "rewards/rejected": -43.19374084472656,
      "step": 3030
    },
    {
      "epoch": 1.6243518983107543,
      "grad_norm": 9.665003025245416e-08,
      "learning_rate": 5.147931662540144e-07,
      "logits/chosen": 8.516685485839844,
      "logits/rejected": 8.1073637008667,
      "logps/chosen": -40.8177375793457,
      "logps/rejected": -42.197906494140625,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -40.8177375793457,
      "rewards/margins": 1.3801673650741577,
      "rewards/rejected": -42.197906494140625,
      "step": 3035
    },
    {
      "epoch": 1.6270279310921558,
      "grad_norm": 8.338258397717798e-09,
      "learning_rate": 5.132363761319449e-07,
      "logits/chosen": 8.531752586364746,
      "logits/rejected": 8.431139945983887,
      "logps/chosen": -41.4906120300293,
      "logps/rejected": -42.645660400390625,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.4906120300293,
      "rewards/margins": 1.1550530195236206,
      "rewards/rejected": -42.645660400390625,
      "step": 3040
    },
    {
      "epoch": 1.6297039638735575,
      "grad_norm": 1.982968074960198e-07,
      "learning_rate": 5.116794575904962e-07,
      "logits/chosen": 8.606687545776367,
      "logits/rejected": 8.396788597106934,
      "logps/chosen": -42.18719482421875,
      "logps/rejected": -42.94538116455078,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -42.18719482421875,
      "rewards/margins": 0.7581872344017029,
      "rewards/rejected": -42.94538116455078,
      "step": 3045
    },
    {
      "epoch": 1.632379996654959,
      "grad_norm": 6.090119324707206e-08,
      "learning_rate": 5.101224257348987e-07,
      "logits/chosen": 8.812459945678711,
      "logits/rejected": 8.619714736938477,
      "logps/chosen": -41.88135528564453,
      "logps/rejected": -42.764671325683594,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.88135528564453,
      "rewards/margins": 0.8833135366439819,
      "rewards/rejected": -42.764671325683594,
      "step": 3050
    },
    {
      "epoch": 1.6350560294363605,
      "grad_norm": 5.954460611989324e-07,
      "learning_rate": 5.085652956714823e-07,
      "logits/chosen": 9.324695587158203,
      "logits/rejected": 8.874685287475586,
      "logps/chosen": -42.326904296875,
      "logps/rejected": -43.86783981323242,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.326904296875,
      "rewards/margins": 1.5409380197525024,
      "rewards/rejected": -43.86783981323242,
      "step": 3055
    },
    {
      "epoch": 1.6377320622177622,
      "grad_norm": 5.783703876784812e-06,
      "learning_rate": 5.070080825075298e-07,
      "logits/chosen": 8.880809783935547,
      "logits/rejected": 8.35167121887207,
      "logps/chosen": -41.295589447021484,
      "logps/rejected": -42.723167419433594,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.295589447021484,
      "rewards/margins": 1.4275810718536377,
      "rewards/rejected": -42.723167419433594,
      "step": 3060
    },
    {
      "epoch": 1.6404080949991637,
      "grad_norm": 2.231494740385337e-11,
      "learning_rate": 5.0545080135113e-07,
      "logits/chosen": 8.833189010620117,
      "logits/rejected": 8.609594345092773,
      "logps/chosen": -43.12578582763672,
      "logps/rejected": -44.05109405517578,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.12578582763672,
      "rewards/margins": 0.9253073930740356,
      "rewards/rejected": -44.05109405517578,
      "step": 3065
    },
    {
      "epoch": 1.6430841277805652,
      "grad_norm": 2.7928006753618e-07,
      "learning_rate": 5.038934673110316e-07,
      "logits/chosen": 8.685979843139648,
      "logits/rejected": 8.558248519897461,
      "logps/chosen": -41.523372650146484,
      "logps/rejected": -42.68616485595703,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -41.523372650146484,
      "rewards/margins": 1.1627862453460693,
      "rewards/rejected": -42.68616485595703,
      "step": 3070
    },
    {
      "epoch": 1.645760160561967,
      "grad_norm": 4.930661480298808e-08,
      "learning_rate": 5.023360954964963e-07,
      "logits/chosen": 8.807648658752441,
      "logits/rejected": 8.644891738891602,
      "logps/chosen": -41.85204315185547,
      "logps/rejected": -42.57401657104492,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -41.85204315185547,
      "rewards/margins": 0.7219740152359009,
      "rewards/rejected": -42.57401657104492,
      "step": 3075
    },
    {
      "epoch": 1.6484361933433684,
      "grad_norm": 1.9041645577364865e-07,
      "learning_rate": 5.007787010171524e-07,
      "logits/chosen": 8.918360710144043,
      "logits/rejected": 8.456040382385254,
      "logps/chosen": -41.612388610839844,
      "logps/rejected": -42.90841293334961,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.612388610839844,
      "rewards/margins": 1.2960256338119507,
      "rewards/rejected": -42.90841293334961,
      "step": 3080
    },
    {
      "epoch": 1.65111222612477,
      "grad_norm": 1.4511974835885142e-10,
      "learning_rate": 4.992212989828477e-07,
      "logits/chosen": 8.849992752075195,
      "logits/rejected": 8.772486686706543,
      "logps/chosen": -43.92622756958008,
      "logps/rejected": -43.822044372558594,
      "loss": 0.0,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -43.92622756958008,
      "rewards/margins": -0.10418035089969635,
      "rewards/rejected": -43.822044372558594,
      "step": 3085
    },
    {
      "epoch": 1.6537882589061716,
      "grad_norm": 2.290444107710993e-07,
      "learning_rate": 4.976639045035036e-07,
      "logits/chosen": 8.790390968322754,
      "logits/rejected": 8.568853378295898,
      "logps/chosen": -43.10242462158203,
      "logps/rejected": -44.56400680541992,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.10242462158203,
      "rewards/margins": 1.461582064628601,
      "rewards/rejected": -44.56400680541992,
      "step": 3090
    },
    {
      "epoch": 1.6564642916875731,
      "grad_norm": 0.255098012707323,
      "learning_rate": 4.961065326889683e-07,
      "logits/chosen": 8.562554359436035,
      "logits/rejected": 8.066210746765137,
      "logps/chosen": -41.56775665283203,
      "logps/rejected": -43.64479064941406,
      "loss": 0.0015,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -41.56775665283203,
      "rewards/margins": 2.0770323276519775,
      "rewards/rejected": -43.64479064941406,
      "step": 3095
    },
    {
      "epoch": 1.6591403244689746,
      "grad_norm": 1.558531275998947e-11,
      "learning_rate": 4.9454919864887e-07,
      "logits/chosen": 8.840633392333984,
      "logits/rejected": 8.545955657958984,
      "logps/chosen": -42.236541748046875,
      "logps/rejected": -43.12220001220703,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.236541748046875,
      "rewards/margins": 0.8856558799743652,
      "rewards/rejected": -43.12220001220703,
      "step": 3100
    },
    {
      "epoch": 1.6618163572503764,
      "grad_norm": 3.7567761127007805e-11,
      "learning_rate": 4.929919174924701e-07,
      "logits/chosen": 9.287446022033691,
      "logits/rejected": 8.558603286743164,
      "logps/chosen": -41.95670700073242,
      "logps/rejected": -43.232784271240234,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.95670700073242,
      "rewards/margins": 1.2760719060897827,
      "rewards/rejected": -43.232784271240234,
      "step": 3105
    },
    {
      "epoch": 1.6644923900317778,
      "grad_norm": 2.530764878293562e-10,
      "learning_rate": 4.914347043285177e-07,
      "logits/chosen": 8.711963653564453,
      "logits/rejected": 8.42900276184082,
      "logps/chosen": -41.76972579956055,
      "logps/rejected": -43.00391387939453,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -41.76972579956055,
      "rewards/margins": 1.2341920137405396,
      "rewards/rejected": -43.00391387939453,
      "step": 3110
    },
    {
      "epoch": 1.6671684228131793,
      "grad_norm": 2.468041771541444e-06,
      "learning_rate": 4.898775742651013e-07,
      "logits/chosen": 8.626786231994629,
      "logits/rejected": 8.341765403747559,
      "logps/chosen": -42.278099060058594,
      "logps/rejected": -43.66221237182617,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.278099060058594,
      "rewards/margins": 1.3841116428375244,
      "rewards/rejected": -43.66221237182617,
      "step": 3115
    },
    {
      "epoch": 1.669844455594581,
      "grad_norm": 4.222630235347758e-07,
      "learning_rate": 4.883205424095037e-07,
      "logits/chosen": 9.043296813964844,
      "logits/rejected": 8.531747817993164,
      "logps/chosen": -41.197139739990234,
      "logps/rejected": -43.73224639892578,
      "loss": 0.0,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -41.197139739990234,
      "rewards/margins": 2.5351052284240723,
      "rewards/rejected": -43.73224639892578,
      "step": 3120
    },
    {
      "epoch": 1.6725204883759828,
      "grad_norm": 0.007342146661988087,
      "learning_rate": 4.86763623868055e-07,
      "logits/chosen": 9.002099990844727,
      "logits/rejected": 8.551979064941406,
      "logps/chosen": -41.3134651184082,
      "logps/rejected": -42.92992401123047,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.3134651184082,
      "rewards/margins": 1.6164617538452148,
      "rewards/rejected": -42.92992401123047,
      "step": 3125
    },
    {
      "epoch": 1.675196521157384,
      "grad_norm": 1.2857323561762575e-05,
      "learning_rate": 4.852068337459856e-07,
      "logits/chosen": 8.533288955688477,
      "logits/rejected": 7.754534721374512,
      "logps/chosen": -40.462135314941406,
      "logps/rejected": -42.256534576416016,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -40.462135314941406,
      "rewards/margins": 1.794398307800293,
      "rewards/rejected": -42.256534576416016,
      "step": 3130
    },
    {
      "epoch": 1.6778725539387858,
      "grad_norm": 2.3326694082558964e-11,
      "learning_rate": 4.8365018714728e-07,
      "logits/chosen": 8.699747085571289,
      "logits/rejected": 8.487561225891113,
      "logps/chosen": -40.05158233642578,
      "logps/rejected": -42.60285186767578,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -40.05158233642578,
      "rewards/margins": 2.5512731075286865,
      "rewards/rejected": -42.60285186767578,
      "step": 3135
    },
    {
      "epoch": 1.6805485867201875,
      "grad_norm": 0.005859951953461854,
      "learning_rate": 4.820936991745304e-07,
      "logits/chosen": 9.187182426452637,
      "logits/rejected": 8.881650924682617,
      "logps/chosen": -41.60440444946289,
      "logps/rejected": -43.28955841064453,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.60440444946289,
      "rewards/margins": 1.6851520538330078,
      "rewards/rejected": -43.28955841064453,
      "step": 3140
    },
    {
      "epoch": 1.6832246195015887,
      "grad_norm": 2.798701663725085e-05,
      "learning_rate": 4.8053738492879e-07,
      "logits/chosen": 8.451224327087402,
      "logits/rejected": 8.048616409301758,
      "logps/chosen": -41.843589782714844,
      "logps/rejected": -42.90123748779297,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -41.843589782714844,
      "rewards/margins": 1.0576448440551758,
      "rewards/rejected": -42.90123748779297,
      "step": 3145
    },
    {
      "epoch": 1.6859006522829905,
      "grad_norm": 0.28570398984484674,
      "learning_rate": 4.789812595094265e-07,
      "logits/chosen": 8.953600883483887,
      "logits/rejected": 8.573225021362305,
      "logps/chosen": -40.707984924316406,
      "logps/rejected": -41.96226119995117,
      "loss": 0.001,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -40.707984924316406,
      "rewards/margins": 1.2542773485183716,
      "rewards/rejected": -41.96226119995117,
      "step": 3150
    },
    {
      "epoch": 1.6885766850643922,
      "grad_norm": 5.159242648943684e-09,
      "learning_rate": 4.774253380139752e-07,
      "logits/chosen": 9.168846130371094,
      "logits/rejected": 8.793378829956055,
      "logps/chosen": -42.01328659057617,
      "logps/rejected": -43.22798538208008,
      "loss": 0.0006,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.01328659057617,
      "rewards/margins": 1.2146974802017212,
      "rewards/rejected": -43.22798538208008,
      "step": 3155
    },
    {
      "epoch": 1.6912527178457935,
      "grad_norm": 2.518944929336543e-08,
      "learning_rate": 4.758696355379936e-07,
      "logits/chosen": 8.72606372833252,
      "logits/rejected": 8.94891357421875,
      "logps/chosen": -42.74934768676758,
      "logps/rejected": -43.51082992553711,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -42.74934768676758,
      "rewards/margins": 0.7614868879318237,
      "rewards/rejected": -43.51082992553711,
      "step": 3160
    },
    {
      "epoch": 1.6939287506271952,
      "grad_norm": 5.6925332966308006e-08,
      "learning_rate": 4.743141671749138e-07,
      "logits/chosen": 9.195821762084961,
      "logits/rejected": 8.750822067260742,
      "logps/chosen": -40.8465576171875,
      "logps/rejected": -41.42811965942383,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -40.8465576171875,
      "rewards/margins": 0.58156818151474,
      "rewards/rejected": -41.42811965942383,
      "step": 3165
    },
    {
      "epoch": 1.6966047834085969,
      "grad_norm": 4.879495282623498e-06,
      "learning_rate": 4.727589480158968e-07,
      "logits/chosen": 9.07165813446045,
      "logits/rejected": 8.73912239074707,
      "logps/chosen": -42.56293869018555,
      "logps/rejected": -44.23527526855469,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.56293869018555,
      "rewards/margins": 1.6723308563232422,
      "rewards/rejected": -44.23527526855469,
      "step": 3170
    },
    {
      "epoch": 1.6992808161899984,
      "grad_norm": 3.5730025945502565e-06,
      "learning_rate": 4.712039931496855e-07,
      "logits/chosen": 8.848831176757812,
      "logits/rejected": 8.2403564453125,
      "logps/chosen": -42.49853515625,
      "logps/rejected": -43.76799011230469,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.49853515625,
      "rewards/margins": 1.2694507837295532,
      "rewards/rejected": -43.76799011230469,
      "step": 3175
    },
    {
      "epoch": 1.7019568489713999,
      "grad_norm": 1.1691293002224624e-08,
      "learning_rate": 4.6964931766245905e-07,
      "logits/chosen": 8.707847595214844,
      "logits/rejected": 8.587430953979492,
      "logps/chosen": -42.651466369628906,
      "logps/rejected": -42.958961486816406,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.651466369628906,
      "rewards/margins": 0.30750173330307007,
      "rewards/rejected": -42.958961486816406,
      "step": 3180
    },
    {
      "epoch": 1.7046328817528016,
      "grad_norm": 6.2873077178589455e-06,
      "learning_rate": 4.6809493663768575e-07,
      "logits/chosen": 8.599573135375977,
      "logits/rejected": 8.256446838378906,
      "logps/chosen": -43.25920104980469,
      "logps/rejected": -43.644981384277344,
      "loss": 0.0,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -43.25920104980469,
      "rewards/margins": 0.3857851028442383,
      "rewards/rejected": -43.644981384277344,
      "step": 3185
    },
    {
      "epoch": 1.707308914534203,
      "grad_norm": 3.4720259755373406e-10,
      "learning_rate": 4.6654086515597716e-07,
      "logits/chosen": 9.07559585571289,
      "logits/rejected": 8.394869804382324,
      "logps/chosen": -41.30718994140625,
      "logps/rejected": -43.453346252441406,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.30718994140625,
      "rewards/margins": 2.1461596488952637,
      "rewards/rejected": -43.453346252441406,
      "step": 3190
    },
    {
      "epoch": 1.7099849473156046,
      "grad_norm": 0.15178583958928252,
      "learning_rate": 4.6498711829494154e-07,
      "logits/chosen": 8.776254653930664,
      "logits/rejected": 8.530635833740234,
      "logps/chosen": -41.66889953613281,
      "logps/rejected": -42.503753662109375,
      "loss": 0.0006,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.66889953613281,
      "rewards/margins": 0.8348552584648132,
      "rewards/rejected": -42.503753662109375,
      "step": 3195
    },
    {
      "epoch": 1.7126609800970063,
      "grad_norm": 1.4032553875680568e-10,
      "learning_rate": 4.6343371112903777e-07,
      "logits/chosen": 8.957771301269531,
      "logits/rejected": 8.334024429321289,
      "logps/chosen": -42.99253463745117,
      "logps/rejected": -43.53425216674805,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.99253463745117,
      "rewards/margins": 0.5417177677154541,
      "rewards/rejected": -43.53425216674805,
      "step": 3200
    },
    {
      "epoch": 1.7126609800970063,
      "eval_logits/chosen": 7.346385478973389,
      "eval_logits/rejected": 7.090656757354736,
      "eval_logps/chosen": -41.73725509643555,
      "eval_logps/rejected": -42.9487419128418,
      "eval_loss": 2.6891856919064594e-07,
      "eval_rewards/accuracies": 0.5675074458122253,
      "eval_rewards/chosen": -41.73725509643555,
      "eval_rewards/margins": 1.211488127708435,
      "eval_rewards/rejected": -42.9487419128418,
      "eval_runtime": 40.2607,
      "eval_samples_per_second": 33.407,
      "eval_steps_per_second": 8.37,
      "step": 3200
    },
    {
      "epoch": 1.7153370128784078,
      "grad_norm": 8.079141973703885e-09,
      "learning_rate": 4.618806587294291e-07,
      "logits/chosen": 8.810081481933594,
      "logits/rejected": 8.665169715881348,
      "logps/chosen": -41.76563262939453,
      "logps/rejected": -43.30455780029297,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.76563262939453,
      "rewards/margins": 1.5389270782470703,
      "rewards/rejected": -43.30455780029297,
      "step": 3205
    },
    {
      "epoch": 1.7180130456598093,
      "grad_norm": 0.0005438265096157191,
      "learning_rate": 4.603279761638365e-07,
      "logits/chosen": 8.74320125579834,
      "logits/rejected": 8.397790908813477,
      "logps/chosen": -42.464683532714844,
      "logps/rejected": -43.62255096435547,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.464683532714844,
      "rewards/margins": 1.1578786373138428,
      "rewards/rejected": -43.62255096435547,
      "step": 3210
    },
    {
      "epoch": 1.720689078441211,
      "grad_norm": 9.406511668348795e-09,
      "learning_rate": 4.5877567849639315e-07,
      "logits/chosen": 9.017813682556152,
      "logits/rejected": 8.773551940917969,
      "logps/chosen": -42.185218811035156,
      "logps/rejected": -43.79137420654297,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.185218811035156,
      "rewards/margins": 1.606148362159729,
      "rewards/rejected": -43.79137420654297,
      "step": 3215
    },
    {
      "epoch": 1.7233651112226125,
      "grad_norm": 9.8056721958456e-10,
      "learning_rate": 4.572237807874979e-07,
      "logits/chosen": 9.223387718200684,
      "logits/rejected": 8.160594940185547,
      "logps/chosen": -41.920066833496094,
      "logps/rejected": -43.96269989013672,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.920066833496094,
      "rewards/margins": 2.0426363945007324,
      "rewards/rejected": -43.96269989013672,
      "step": 3220
    },
    {
      "epoch": 1.726041144004014,
      "grad_norm": 1.674313073824762e-07,
      "learning_rate": 4.5567229809366895e-07,
      "logits/chosen": 8.829940795898438,
      "logits/rejected": 8.46506118774414,
      "logps/chosen": -42.271976470947266,
      "logps/rejected": -43.912628173828125,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.271976470947266,
      "rewards/margins": 1.6406564712524414,
      "rewards/rejected": -43.912628173828125,
      "step": 3225
    },
    {
      "epoch": 1.7287171767854157,
      "grad_norm": 1.4514295156358748e-07,
      "learning_rate": 4.541212454673984e-07,
      "logits/chosen": 8.992741584777832,
      "logits/rejected": 8.347302436828613,
      "logps/chosen": -42.49037551879883,
      "logps/rejected": -44.81675720214844,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.49037551879883,
      "rewards/margins": 2.3263790607452393,
      "rewards/rejected": -44.81675720214844,
      "step": 3230
    },
    {
      "epoch": 1.7313932095668172,
      "grad_norm": 7.571709446908636e-08,
      "learning_rate": 4.525706379570055e-07,
      "logits/chosen": 9.3079252243042,
      "logits/rejected": 8.997198104858398,
      "logps/chosen": -41.79115676879883,
      "logps/rejected": -43.184913635253906,
      "loss": 0.0002,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.79115676879883,
      "rewards/margins": 1.3937630653381348,
      "rewards/rejected": -43.184913635253906,
      "step": 3235
    },
    {
      "epoch": 1.7340692423482187,
      "grad_norm": 2.9759526482329537e-07,
      "learning_rate": 4.510204906064911e-07,
      "logits/chosen": 8.885148048400879,
      "logits/rejected": 8.273990631103516,
      "logps/chosen": -41.427982330322266,
      "logps/rejected": -43.95401382446289,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -41.427982330322266,
      "rewards/margins": 2.5260348320007324,
      "rewards/rejected": -43.95401382446289,
      "step": 3240
    },
    {
      "epoch": 1.7367452751296204,
      "grad_norm": 2.6378500244696348e-05,
      "learning_rate": 4.4947081845539177e-07,
      "logits/chosen": 9.159140586853027,
      "logits/rejected": 8.722005844116211,
      "logps/chosen": -42.572532653808594,
      "logps/rejected": -43.89950180053711,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.572532653808594,
      "rewards/margins": 1.326970100402832,
      "rewards/rejected": -43.89950180053711,
      "step": 3245
    },
    {
      "epoch": 1.739421307911022,
      "grad_norm": 3.3489562044606567e-06,
      "learning_rate": 4.479216365386333e-07,
      "logits/chosen": 8.948904037475586,
      "logits/rejected": 8.283456802368164,
      "logps/chosen": -41.316383361816406,
      "logps/rejected": -43.39640808105469,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -41.316383361816406,
      "rewards/margins": 2.0800294876098633,
      "rewards/rejected": -43.39640808105469,
      "step": 3250
    },
    {
      "epoch": 1.7420973406924234,
      "grad_norm": 1.0662338456938564e-08,
      "learning_rate": 4.4637295988638555e-07,
      "logits/chosen": 9.030210494995117,
      "logits/rejected": 8.936519622802734,
      "logps/chosen": -41.05961990356445,
      "logps/rejected": -43.70038604736328,
      "loss": 0.0,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -41.05961990356445,
      "rewards/margins": 2.6407690048217773,
      "rewards/rejected": -43.70038604736328,
      "step": 3255
    },
    {
      "epoch": 1.744773373473825,
      "grad_norm": 3.379130417332738e-05,
      "learning_rate": 4.4482480352391623e-07,
      "logits/chosen": 8.51530647277832,
      "logits/rejected": 8.508938789367676,
      "logps/chosen": -41.11503219604492,
      "logps/rejected": -42.8921012878418,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -41.11503219604492,
      "rewards/margins": 1.7770659923553467,
      "rewards/rejected": -42.8921012878418,
      "step": 3260
    },
    {
      "epoch": 1.7474494062552266,
      "grad_norm": 1.0016449783260726e-11,
      "learning_rate": 4.4327718247144507e-07,
      "logits/chosen": 8.70158576965332,
      "logits/rejected": 8.335428237915039,
      "logps/chosen": -42.67002868652344,
      "logps/rejected": -43.72126007080078,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.67002868652344,
      "rewards/margins": 1.0512341260910034,
      "rewards/rejected": -43.72126007080078,
      "step": 3265
    },
    {
      "epoch": 1.750125439036628,
      "grad_norm": 1.2840676079622715e-05,
      "learning_rate": 4.417301117439984e-07,
      "logits/chosen": 9.127232551574707,
      "logits/rejected": 8.941888809204102,
      "logps/chosen": -41.003997802734375,
      "logps/rejected": -42.68315505981445,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -41.003997802734375,
      "rewards/margins": 1.6791555881500244,
      "rewards/rejected": -42.68315505981445,
      "step": 3270
    },
    {
      "epoch": 1.7528014718180298,
      "grad_norm": 1.628372060347152e-11,
      "learning_rate": 4.401836063512631e-07,
      "logits/chosen": 9.382959365844727,
      "logits/rejected": 8.315888404846191,
      "logps/chosen": -43.123863220214844,
      "logps/rejected": -44.72638702392578,
      "loss": 0.0003,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.123863220214844,
      "rewards/margins": 1.6025243997573853,
      "rewards/rejected": -44.72638702392578,
      "step": 3275
    },
    {
      "epoch": 1.7554775045994313,
      "grad_norm": 2.5167838552479552e-12,
      "learning_rate": 4.386376812974413e-07,
      "logits/chosen": 9.1256742477417,
      "logits/rejected": 9.009529113769531,
      "logps/chosen": -44.24250030517578,
      "logps/rejected": -45.69837188720703,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.24250030517578,
      "rewards/margins": 1.455871820449829,
      "rewards/rejected": -45.69837188720703,
      "step": 3280
    },
    {
      "epoch": 1.7581535373808328,
      "grad_norm": 2.2631180011183867e-07,
      "learning_rate": 4.370923515811048e-07,
      "logits/chosen": 9.232044219970703,
      "logits/rejected": 8.624814987182617,
      "logps/chosen": -42.31184768676758,
      "logps/rejected": -43.475624084472656,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.31184768676758,
      "rewards/margins": 1.1637694835662842,
      "rewards/rejected": -43.475624084472656,
      "step": 3285
    },
    {
      "epoch": 1.7608295701622345,
      "grad_norm": 1.660488176309308e-07,
      "learning_rate": 4.35547632195049e-07,
      "logits/chosen": 8.911782264709473,
      "logits/rejected": 8.576136589050293,
      "logps/chosen": -43.006004333496094,
      "logps/rejected": -43.88723373413086,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.006004333496094,
      "rewards/margins": 0.8812358975410461,
      "rewards/rejected": -43.88723373413086,
      "step": 3290
    },
    {
      "epoch": 1.763505602943636,
      "grad_norm": 4.725766750356587e-11,
      "learning_rate": 4.340035381261484e-07,
      "logits/chosen": 9.171948432922363,
      "logits/rejected": 8.768861770629883,
      "logps/chosen": -43.28553009033203,
      "logps/rejected": -44.162353515625,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -43.28553009033203,
      "rewards/margins": 0.8768193125724792,
      "rewards/rejected": -44.162353515625,
      "step": 3295
    },
    {
      "epoch": 1.7661816357250375,
      "grad_norm": 1.3895684667074638e-08,
      "learning_rate": 4.324600843552104e-07,
      "logits/chosen": 9.119513511657715,
      "logits/rejected": 8.699051856994629,
      "logps/chosen": -42.70151901245117,
      "logps/rejected": -43.992042541503906,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.70151901245117,
      "rewards/margins": 1.2905250787734985,
      "rewards/rejected": -43.992042541503906,
      "step": 3300
    },
    {
      "epoch": 1.7688576685064392,
      "grad_norm": 0.0001982070200975259,
      "learning_rate": 4.309172858568302e-07,
      "logits/chosen": 8.971323013305664,
      "logits/rejected": 8.495769500732422,
      "logps/chosen": -42.44605255126953,
      "logps/rejected": -43.94598388671875,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.44605255126953,
      "rewards/margins": 1.4999330043792725,
      "rewards/rejected": -43.94598388671875,
      "step": 3305
    },
    {
      "epoch": 1.771533701287841,
      "grad_norm": 3.528499885958161e-10,
      "learning_rate": 4.293751575992455e-07,
      "logits/chosen": 8.718727111816406,
      "logits/rejected": 8.726655960083008,
      "logps/chosen": -42.47847366333008,
      "logps/rejected": -43.79438400268555,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.47847366333008,
      "rewards/margins": 1.3159105777740479,
      "rewards/rejected": -43.79438400268555,
      "step": 3310
    },
    {
      "epoch": 1.7742097340692422,
      "grad_norm": 1.3055398504239128e-05,
      "learning_rate": 4.278337145441916e-07,
      "logits/chosen": 9.055317878723145,
      "logits/rejected": 8.470276832580566,
      "logps/chosen": -41.76118850708008,
      "logps/rejected": -42.783653259277344,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.76118850708008,
      "rewards/margins": 1.0224684476852417,
      "rewards/rejected": -42.783653259277344,
      "step": 3315
    },
    {
      "epoch": 1.776885766850644,
      "grad_norm": 3.468722492300036e-08,
      "learning_rate": 4.262929716467556e-07,
      "logits/chosen": 9.097979545593262,
      "logits/rejected": 8.224995613098145,
      "logps/chosen": -42.98595428466797,
      "logps/rejected": -44.47277069091797,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.98595428466797,
      "rewards/margins": 1.4868137836456299,
      "rewards/rejected": -44.47277069091797,
      "step": 3320
    },
    {
      "epoch": 1.7795617996320456,
      "grad_norm": 3.573175488908807e-11,
      "learning_rate": 4.247529438552321e-07,
      "logits/chosen": 9.045320510864258,
      "logits/rejected": 8.452642440795898,
      "logps/chosen": -41.813297271728516,
      "logps/rejected": -42.73538589477539,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -41.813297271728516,
      "rewards/margins": 0.9220888018608093,
      "rewards/rejected": -42.73538589477539,
      "step": 3325
    },
    {
      "epoch": 1.782237832413447,
      "grad_norm": 7.361206936683324e-07,
      "learning_rate": 4.232136461109773e-07,
      "logits/chosen": 9.049830436706543,
      "logits/rejected": 8.575239181518555,
      "logps/chosen": -42.71787643432617,
      "logps/rejected": -43.28335952758789,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.71787643432617,
      "rewards/margins": 0.5654757618904114,
      "rewards/rejected": -43.28335952758789,
      "step": 3330
    },
    {
      "epoch": 1.7849138651948486,
      "grad_norm": 0.08316199103227653,
      "learning_rate": 4.216750933482646e-07,
      "logits/chosen": 8.697958946228027,
      "logits/rejected": 8.333512306213379,
      "logps/chosen": -42.26152801513672,
      "logps/rejected": -43.87995910644531,
      "loss": 0.0007,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.26152801513672,
      "rewards/margins": 1.6184282302856445,
      "rewards/rejected": -43.87995910644531,
      "step": 3335
    },
    {
      "epoch": 1.7875898979762503,
      "grad_norm": 2.0258879061185094e-07,
      "learning_rate": 4.2013730049413986e-07,
      "logits/chosen": 8.910024642944336,
      "logits/rejected": 8.129594802856445,
      "logps/chosen": -41.321998596191406,
      "logps/rejected": -43.52183151245117,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -41.321998596191406,
      "rewards/margins": 2.19983172416687,
      "rewards/rejected": -43.52183151245117,
      "step": 3340
    },
    {
      "epoch": 1.7902659307576518,
      "grad_norm": 4.7136050765706833e-07,
      "learning_rate": 4.1860028246827594e-07,
      "logits/chosen": 8.940666198730469,
      "logits/rejected": 8.155450820922852,
      "logps/chosen": -43.48246765136719,
      "logps/rejected": -44.367820739746094,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.48246765136719,
      "rewards/margins": 0.8853505849838257,
      "rewards/rejected": -44.367820739746094,
      "step": 3345
    },
    {
      "epoch": 1.7929419635390533,
      "grad_norm": 1.1149191386631303e-09,
      "learning_rate": 4.170640541828285e-07,
      "logits/chosen": 9.452614784240723,
      "logits/rejected": 9.082446098327637,
      "logps/chosen": -42.882118225097656,
      "logps/rejected": -43.46571350097656,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.882118225097656,
      "rewards/margins": 0.5835954546928406,
      "rewards/rejected": -43.46571350097656,
      "step": 3350
    },
    {
      "epoch": 1.795617996320455,
      "grad_norm": 2.0719172991565895e-10,
      "learning_rate": 4.1552863054229116e-07,
      "logits/chosen": 8.989095687866211,
      "logits/rejected": 8.857994079589844,
      "logps/chosen": -42.419124603271484,
      "logps/rejected": -44.075111389160156,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -42.419124603271484,
      "rewards/margins": 1.6559826135635376,
      "rewards/rejected": -44.075111389160156,
      "step": 3355
    },
    {
      "epoch": 1.7982940291018565,
      "grad_norm": 2.9478594166596984e-05,
      "learning_rate": 4.139940264433508e-07,
      "logits/chosen": 9.005237579345703,
      "logits/rejected": 8.315083503723145,
      "logps/chosen": -43.112281799316406,
      "logps/rejected": -44.65152359008789,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.112281799316406,
      "rewards/margins": 1.5392391681671143,
      "rewards/rejected": -44.65152359008789,
      "step": 3360
    },
    {
      "epoch": 1.800970061883258,
      "grad_norm": 1.87954822593384e-09,
      "learning_rate": 4.1246025677474303e-07,
      "logits/chosen": 9.404446601867676,
      "logits/rejected": 8.712800979614258,
      "logps/chosen": -42.869224548339844,
      "logps/rejected": -44.20194625854492,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.869224548339844,
      "rewards/margins": 1.3327208757400513,
      "rewards/rejected": -44.20194625854492,
      "step": 3365
    },
    {
      "epoch": 1.8036460946646597,
      "grad_norm": 7.400907333812676e-10,
      "learning_rate": 4.10927336417108e-07,
      "logits/chosen": 9.088297843933105,
      "logits/rejected": 8.417060852050781,
      "logps/chosen": -42.23302459716797,
      "logps/rejected": -43.39586639404297,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.23302459716797,
      "rewards/margins": 1.1628367900848389,
      "rewards/rejected": -43.39586639404297,
      "step": 3370
    },
    {
      "epoch": 1.8063221274460612,
      "grad_norm": 5.508248979305815e-07,
      "learning_rate": 4.093952802428457e-07,
      "logits/chosen": 8.810931205749512,
      "logits/rejected": 8.732718467712402,
      "logps/chosen": -42.82012939453125,
      "logps/rejected": -43.050628662109375,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -42.82012939453125,
      "rewards/margins": 0.23050299286842346,
      "rewards/rejected": -43.050628662109375,
      "step": 3375
    },
    {
      "epoch": 1.8089981602274627,
      "grad_norm": 2.867034886440815e-11,
      "learning_rate": 4.0786410311597184e-07,
      "logits/chosen": 9.028701782226562,
      "logits/rejected": 8.337693214416504,
      "logps/chosen": -41.94820022583008,
      "logps/rejected": -42.89849090576172,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.94820022583008,
      "rewards/margins": 0.9502902030944824,
      "rewards/rejected": -42.89849090576172,
      "step": 3380
    },
    {
      "epoch": 1.8116741930088645,
      "grad_norm": 7.006047306199805e-08,
      "learning_rate": 4.063338198919737e-07,
      "logits/chosen": 9.156309127807617,
      "logits/rejected": 9.12796688079834,
      "logps/chosen": -42.02606964111328,
      "logps/rejected": -42.69218826293945,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -42.02606964111328,
      "rewards/margins": 0.6661237478256226,
      "rewards/rejected": -42.69218826293945,
      "step": 3385
    },
    {
      "epoch": 1.814350225790266,
      "grad_norm": 6.430880006939129e-13,
      "learning_rate": 4.0480444541766575e-07,
      "logits/chosen": 9.370391845703125,
      "logits/rejected": 8.873638153076172,
      "logps/chosen": -43.99597930908203,
      "logps/rejected": -45.247283935546875,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.99597930908203,
      "rewards/margins": 1.2513020038604736,
      "rewards/rejected": -45.247283935546875,
      "step": 3390
    },
    {
      "epoch": 1.8170262585716674,
      "grad_norm": 3.104439514603226e-10,
      "learning_rate": 4.0327599453104606e-07,
      "logits/chosen": 9.182734489440918,
      "logits/rejected": 8.787361145019531,
      "logps/chosen": -41.466064453125,
      "logps/rejected": -43.58794403076172,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.466064453125,
      "rewards/margins": 2.1218810081481934,
      "rewards/rejected": -43.58794403076172,
      "step": 3395
    },
    {
      "epoch": 1.8197022913530692,
      "grad_norm": 7.199554098129442e-08,
      "learning_rate": 4.017484820611514e-07,
      "logits/chosen": 8.849800109863281,
      "logits/rejected": 8.342534065246582,
      "logps/chosen": -42.05352020263672,
      "logps/rejected": -43.215274810791016,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.05352020263672,
      "rewards/margins": 1.1617573499679565,
      "rewards/rejected": -43.215274810791016,
      "step": 3400
    },
    {
      "epoch": 1.8223783241344707,
      "grad_norm": 1.0734951778870868e-09,
      "learning_rate": 4.002219228279148e-07,
      "logits/chosen": 8.902721405029297,
      "logits/rejected": 8.534134864807129,
      "logps/chosen": -42.02064895629883,
      "logps/rejected": -43.605655670166016,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.02064895629883,
      "rewards/margins": 1.5850058794021606,
      "rewards/rejected": -43.605655670166016,
      "step": 3405
    },
    {
      "epoch": 1.8250543569158721,
      "grad_norm": 1.655388047059721e-06,
      "learning_rate": 3.9869633164202045e-07,
      "logits/chosen": 9.266801834106445,
      "logits/rejected": 8.2295503616333,
      "logps/chosen": -41.05113983154297,
      "logps/rejected": -42.347869873046875,
      "loss": 0.0001,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -41.05113983154297,
      "rewards/margins": 1.2967309951782227,
      "rewards/rejected": -42.347869873046875,
      "step": 3410
    },
    {
      "epoch": 1.8277303896972739,
      "grad_norm": 5.081088193193098e-07,
      "learning_rate": 3.9717172330476077e-07,
      "logits/chosen": 9.28183364868164,
      "logits/rejected": 8.853327751159668,
      "logps/chosen": -42.637420654296875,
      "logps/rejected": -43.296817779541016,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.637420654296875,
      "rewards/margins": 0.6593946218490601,
      "rewards/rejected": -43.296817779541016,
      "step": 3415
    },
    {
      "epoch": 1.8304064224786754,
      "grad_norm": 7.796237438434782e-07,
      "learning_rate": 3.956481126078927e-07,
      "logits/chosen": 8.982412338256836,
      "logits/rejected": 8.659948348999023,
      "logps/chosen": -43.46683883666992,
      "logps/rejected": -43.597984313964844,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.46683883666992,
      "rewards/margins": 0.13114602863788605,
      "rewards/rejected": -43.597984313964844,
      "step": 3420
    },
    {
      "epoch": 1.8330824552600768,
      "grad_norm": 4.6506318648743666e-10,
      "learning_rate": 3.941255143334937e-07,
      "logits/chosen": 8.835341453552246,
      "logits/rejected": 8.636839866638184,
      "logps/chosen": -42.88193893432617,
      "logps/rejected": -43.88673400878906,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.88193893432617,
      "rewards/margins": 1.0047924518585205,
      "rewards/rejected": -43.88673400878906,
      "step": 3425
    },
    {
      "epoch": 1.8357584880414786,
      "grad_norm": 0.0002164451830728961,
      "learning_rate": 3.9260394325381895e-07,
      "logits/chosen": 8.640530586242676,
      "logits/rejected": 8.23536491394043,
      "logps/chosen": -41.761924743652344,
      "logps/rejected": -43.662899017333984,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -41.761924743652344,
      "rewards/margins": 1.9009778499603271,
      "rewards/rejected": -43.662899017333984,
      "step": 3430
    },
    {
      "epoch": 1.83843452082288,
      "grad_norm": 2.359026558039456e-09,
      "learning_rate": 3.9108341413115784e-07,
      "logits/chosen": 8.89246654510498,
      "logits/rejected": 8.473149299621582,
      "logps/chosen": -41.694679260253906,
      "logps/rejected": -42.69411087036133,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.694679260253906,
      "rewards/margins": 0.9994314908981323,
      "rewards/rejected": -42.69411087036133,
      "step": 3435
    },
    {
      "epoch": 1.8411105536042816,
      "grad_norm": 1.754802318560265e-11,
      "learning_rate": 3.895639417176905e-07,
      "logits/chosen": 9.135909080505371,
      "logits/rejected": 8.80331802368164,
      "logps/chosen": -43.21285629272461,
      "logps/rejected": -44.86457824707031,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.21285629272461,
      "rewards/margins": 1.651720404624939,
      "rewards/rejected": -44.86457824707031,
      "step": 3440
    },
    {
      "epoch": 1.8437865863856833,
      "grad_norm": 5.367246114941999e-07,
      "learning_rate": 3.8804554075534497e-07,
      "logits/chosen": 9.278481483459473,
      "logits/rejected": 8.533929824829102,
      "logps/chosen": -42.80669403076172,
      "logps/rejected": -44.85102081298828,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.80669403076172,
      "rewards/margins": 2.0443243980407715,
      "rewards/rejected": -44.85102081298828,
      "step": 3445
    },
    {
      "epoch": 1.8464626191670848,
      "grad_norm": 1.1818562359115935e-10,
      "learning_rate": 3.8652822597565403e-07,
      "logits/chosen": 9.049799919128418,
      "logits/rejected": 8.538946151733398,
      "logps/chosen": -42.586944580078125,
      "logps/rejected": -44.1765022277832,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -42.586944580078125,
      "rewards/margins": 1.5895531177520752,
      "rewards/rejected": -44.1765022277832,
      "step": 3450
    },
    {
      "epoch": 1.8491386519484863,
      "grad_norm": 1.064864283126891e-09,
      "learning_rate": 3.850120120996123e-07,
      "logits/chosen": 9.163946151733398,
      "logits/rejected": 8.482351303100586,
      "logps/chosen": -42.87347412109375,
      "logps/rejected": -44.42333984375,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -42.87347412109375,
      "rewards/margins": 1.549865961074829,
      "rewards/rejected": -44.42333984375,
      "step": 3455
    },
    {
      "epoch": 1.851814684729888,
      "grad_norm": 1.786478618743661e-10,
      "learning_rate": 3.8349691383753356e-07,
      "logits/chosen": 8.889766693115234,
      "logits/rejected": 8.509964942932129,
      "logps/chosen": -43.50269317626953,
      "logps/rejected": -43.71992874145508,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -43.50269317626953,
      "rewards/margins": 0.2172294557094574,
      "rewards/rejected": -43.71992874145508,
      "step": 3460
    },
    {
      "epoch": 1.8544907175112895,
      "grad_norm": 5.7483043963092624e-08,
      "learning_rate": 3.819829458889078e-07,
      "logits/chosen": 9.000165939331055,
      "logits/rejected": 8.670591354370117,
      "logps/chosen": -41.933738708496094,
      "logps/rejected": -43.36604309082031,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.933738708496094,
      "rewards/margins": 1.432308316230774,
      "rewards/rejected": -43.36604309082031,
      "step": 3465
    },
    {
      "epoch": 1.857166750292691,
      "grad_norm": 5.884668307367081e-07,
      "learning_rate": 3.804701229422585e-07,
      "logits/chosen": 9.101262092590332,
      "logits/rejected": 8.968666076660156,
      "logps/chosen": -42.483253479003906,
      "logps/rejected": -43.319339752197266,
      "loss": 0.0011,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.483253479003906,
      "rewards/margins": 0.8360843658447266,
      "rewards/rejected": -43.319339752197266,
      "step": 3470
    },
    {
      "epoch": 1.8598427830740927,
      "grad_norm": 1.549068521775006e-07,
      "learning_rate": 3.789584596750007e-07,
      "logits/chosen": 8.936643600463867,
      "logits/rejected": 8.723246574401855,
      "logps/chosen": -42.338253021240234,
      "logps/rejected": -43.11792755126953,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.338253021240234,
      "rewards/margins": 0.7796715497970581,
      "rewards/rejected": -43.11792755126953,
      "step": 3475
    },
    {
      "epoch": 1.8625188158554944,
      "grad_norm": 4.6126903926992024e-09,
      "learning_rate": 3.77447970753298e-07,
      "logits/chosen": 8.634635925292969,
      "logits/rejected": 8.493051528930664,
      "logps/chosen": -43.30158233642578,
      "logps/rejected": -43.481624603271484,
      "loss": 0.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -43.30158233642578,
      "rewards/margins": 0.18004471063613892,
      "rewards/rejected": -43.481624603271484,
      "step": 3480
    },
    {
      "epoch": 1.8651948486368957,
      "grad_norm": 8.27703423016579e-06,
      "learning_rate": 3.7593867083192057e-07,
      "logits/chosen": 8.836604118347168,
      "logits/rejected": 8.464637756347656,
      "logps/chosen": -43.114402770996094,
      "logps/rejected": -44.425872802734375,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.114402770996094,
      "rewards/margins": 1.3114697933197021,
      "rewards/rejected": -44.425872802734375,
      "step": 3485
    },
    {
      "epoch": 1.8678708814182974,
      "grad_norm": 4.175922808413568e-08,
      "learning_rate": 3.7443057455410276e-07,
      "logits/chosen": 8.917654037475586,
      "logits/rejected": 8.54902458190918,
      "logps/chosen": -41.48270797729492,
      "logps/rejected": -43.69658660888672,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -41.48270797729492,
      "rewards/margins": 2.213876485824585,
      "rewards/rejected": -43.69658660888672,
      "step": 3490
    },
    {
      "epoch": 1.870546914199699,
      "grad_norm": 2.038271582546916e-12,
      "learning_rate": 3.7292369655140145e-07,
      "logits/chosen": 9.515264511108398,
      "logits/rejected": 8.955233573913574,
      "logps/chosen": -43.51061248779297,
      "logps/rejected": -44.70892333984375,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.51061248779297,
      "rewards/margins": 1.1983106136322021,
      "rewards/rejected": -44.70892333984375,
      "step": 3495
    },
    {
      "epoch": 1.8732229469811004,
      "grad_norm": 5.474106460186873e-06,
      "learning_rate": 3.714180514435534e-07,
      "logits/chosen": 8.58936882019043,
      "logits/rejected": 8.22518253326416,
      "logps/chosen": -42.50271224975586,
      "logps/rejected": -43.35732650756836,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.50271224975586,
      "rewards/margins": 0.8546162843704224,
      "rewards/rejected": -43.35732650756836,
      "step": 3500
    },
    {
      "epoch": 1.875898979762502,
      "grad_norm": 2.0473348054224673e-11,
      "learning_rate": 3.6991365383833426e-07,
      "logits/chosen": 9.062959671020508,
      "logits/rejected": 8.590060234069824,
      "logps/chosen": -42.446285247802734,
      "logps/rejected": -44.069419860839844,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.446285247802734,
      "rewards/margins": 1.6231279373168945,
      "rewards/rejected": -44.069419860839844,
      "step": 3505
    },
    {
      "epoch": 1.8785750125439038,
      "grad_norm": 1.3398116243221677e-07,
      "learning_rate": 3.684105183314162e-07,
      "logits/chosen": 9.43688678741455,
      "logits/rejected": 8.930704116821289,
      "logps/chosen": -43.761383056640625,
      "logps/rejected": -44.966285705566406,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.761383056640625,
      "rewards/margins": 1.204900860786438,
      "rewards/rejected": -44.966285705566406,
      "step": 3510
    },
    {
      "epoch": 1.881251045325305,
      "grad_norm": 8.336248701066181e-08,
      "learning_rate": 3.669086595062263e-07,
      "logits/chosen": 9.222817420959473,
      "logits/rejected": 8.599294662475586,
      "logps/chosen": -41.42193603515625,
      "logps/rejected": -43.319358825683594,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -41.42193603515625,
      "rewards/margins": 1.8974149227142334,
      "rewards/rejected": -43.319358825683594,
      "step": 3515
    },
    {
      "epoch": 1.8839270781067068,
      "grad_norm": 4.99134572985627e-12,
      "learning_rate": 3.654080919338056e-07,
      "logits/chosen": 9.26179313659668,
      "logits/rejected": 8.471601486206055,
      "logps/chosen": -44.294681549072266,
      "logps/rejected": -44.31082534790039,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -44.294681549072266,
      "rewards/margins": 0.016147326678037643,
      "rewards/rejected": -44.31082534790039,
      "step": 3520
    },
    {
      "epoch": 1.8866031108881085,
      "grad_norm": 1.744513978095259e-08,
      "learning_rate": 3.639088301726673e-07,
      "logits/chosen": 8.830717086791992,
      "logits/rejected": 8.207534790039062,
      "logps/chosen": -42.37540054321289,
      "logps/rejected": -43.42636489868164,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.37540054321289,
      "rewards/margins": 1.0509672164916992,
      "rewards/rejected": -43.42636489868164,
      "step": 3525
    },
    {
      "epoch": 1.88927914366951,
      "grad_norm": 1.5537686977102167e-07,
      "learning_rate": 3.624108887686556e-07,
      "logits/chosen": 8.95891284942627,
      "logits/rejected": 8.840760231018066,
      "logps/chosen": -42.549034118652344,
      "logps/rejected": -43.4303092956543,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.549034118652344,
      "rewards/margins": 0.8812732696533203,
      "rewards/rejected": -43.4303092956543,
      "step": 3530
    },
    {
      "epoch": 1.8919551764509115,
      "grad_norm": 3.149750348081701e-09,
      "learning_rate": 3.6091428225480433e-07,
      "logits/chosen": 9.090771675109863,
      "logits/rejected": 8.770219802856445,
      "logps/chosen": -42.282737731933594,
      "logps/rejected": -43.812442779541016,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.282737731933594,
      "rewards/margins": 1.5297024250030518,
      "rewards/rejected": -43.812442779541016,
      "step": 3535
    },
    {
      "epoch": 1.8946312092323132,
      "grad_norm": 1.8547164379454796e-06,
      "learning_rate": 3.5941902515119674e-07,
      "logits/chosen": 9.243340492248535,
      "logits/rejected": 8.353669166564941,
      "logps/chosen": -43.447418212890625,
      "logps/rejected": -43.65853500366211,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.447418212890625,
      "rewards/margins": 0.2111167162656784,
      "rewards/rejected": -43.65853500366211,
      "step": 3540
    },
    {
      "epoch": 1.8973072420137147,
      "grad_norm": 5.766401250556353e-08,
      "learning_rate": 3.5792513196482373e-07,
      "logits/chosen": 9.161626815795898,
      "logits/rejected": 8.19868278503418,
      "logps/chosen": -41.153018951416016,
      "logps/rejected": -44.10417938232422,
      "loss": 0.0,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -41.153018951416016,
      "rewards/margins": 2.9511661529541016,
      "rewards/rejected": -44.10417938232422,
      "step": 3545
    },
    {
      "epoch": 1.8999832747951162,
      "grad_norm": 1.863281745562108e-06,
      "learning_rate": 3.5643261718944346e-07,
      "logits/chosen": 8.499422073364258,
      "logits/rejected": 8.118656158447266,
      "logps/chosen": -41.36172103881836,
      "logps/rejected": -42.181983947753906,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.36172103881836,
      "rewards/margins": 0.8202651143074036,
      "rewards/rejected": -42.181983947753906,
      "step": 3550
    },
    {
      "epoch": 1.902659307576518,
      "grad_norm": 3.842174477720752e-11,
      "learning_rate": 3.5494149530544087e-07,
      "logits/chosen": 9.111536979675293,
      "logits/rejected": 8.595871925354004,
      "logps/chosen": -42.133121490478516,
      "logps/rejected": -42.95072937011719,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -42.133121490478516,
      "rewards/margins": 0.8176079988479614,
      "rewards/rejected": -42.95072937011719,
      "step": 3555
    },
    {
      "epoch": 1.9053353403579194,
      "grad_norm": 1.6768610019370401e-09,
      "learning_rate": 3.534517807796871e-07,
      "logits/chosen": 9.257651329040527,
      "logits/rejected": 8.800093650817871,
      "logps/chosen": -42.34269332885742,
      "logps/rejected": -43.44968795776367,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.34269332885742,
      "rewards/margins": 1.1069934368133545,
      "rewards/rejected": -43.44968795776367,
      "step": 3560
    },
    {
      "epoch": 1.908011373139321,
      "grad_norm": 3.061411973573918e-07,
      "learning_rate": 3.519634880653988e-07,
      "logits/chosen": 8.94935417175293,
      "logits/rejected": 8.788725852966309,
      "logps/chosen": -42.50508499145508,
      "logps/rejected": -44.64189910888672,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.50508499145508,
      "rewards/margins": 2.1368160247802734,
      "rewards/rejected": -44.64189910888672,
      "step": 3565
    },
    {
      "epoch": 1.9106874059207226,
      "grad_norm": 7.560747657077877e-09,
      "learning_rate": 3.504766316019987e-07,
      "logits/chosen": 9.354012489318848,
      "logits/rejected": 8.713424682617188,
      "logps/chosen": -43.34133529663086,
      "logps/rejected": -44.46308898925781,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.34133529663086,
      "rewards/margins": 1.121755838394165,
      "rewards/rejected": -44.46308898925781,
      "step": 3570
    },
    {
      "epoch": 1.913363438702124,
      "grad_norm": 1.0318364823111741e-09,
      "learning_rate": 3.489912258149745e-07,
      "logits/chosen": 9.05507755279541,
      "logits/rejected": 8.614204406738281,
      "logps/chosen": -43.152435302734375,
      "logps/rejected": -44.185462951660156,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.152435302734375,
      "rewards/margins": 1.0330299139022827,
      "rewards/rejected": -44.185462951660156,
      "step": 3575
    },
    {
      "epoch": 1.9160394714835256,
      "grad_norm": 4.114034268115957e-09,
      "learning_rate": 3.475072851157397e-07,
      "logits/chosen": 8.823284149169922,
      "logits/rejected": 8.767995834350586,
      "logps/chosen": -42.063812255859375,
      "logps/rejected": -43.452903747558594,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.063812255859375,
      "rewards/margins": 1.389090895652771,
      "rewards/rejected": -43.452903747558594,
      "step": 3580
    },
    {
      "epoch": 1.9187155042649273,
      "grad_norm": 9.86677188072468e-09,
      "learning_rate": 3.460248239014936e-07,
      "logits/chosen": 8.538291931152344,
      "logits/rejected": 8.421481132507324,
      "logps/chosen": -42.41411209106445,
      "logps/rejected": -43.50041580200195,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.41411209106445,
      "rewards/margins": 1.086301565170288,
      "rewards/rejected": -43.50041580200195,
      "step": 3585
    },
    {
      "epoch": 1.9213915370463288,
      "grad_norm": 4.3948647739649184e-09,
      "learning_rate": 3.4454385655508134e-07,
      "logits/chosen": 8.809771537780762,
      "logits/rejected": 8.738973617553711,
      "logps/chosen": -42.37629699707031,
      "logps/rejected": -43.13481140136719,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.37629699707031,
      "rewards/margins": 0.7585160732269287,
      "rewards/rejected": -43.13481140136719,
      "step": 3590
    },
    {
      "epoch": 1.9240675698277303,
      "grad_norm": 5.569565001193057e-10,
      "learning_rate": 3.4306439744485447e-07,
      "logits/chosen": 9.207661628723145,
      "logits/rejected": 8.426923751831055,
      "logps/chosen": -42.792816162109375,
      "logps/rejected": -43.77056121826172,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.792816162109375,
      "rewards/margins": 0.9777439832687378,
      "rewards/rejected": -43.77056121826172,
      "step": 3595
    },
    {
      "epoch": 1.926743602609132,
      "grad_norm": 6.788564572550981e-06,
      "learning_rate": 3.415864609245322e-07,
      "logits/chosen": 8.945123672485352,
      "logits/rejected": 8.362665176391602,
      "logps/chosen": -43.345211029052734,
      "logps/rejected": -44.7468147277832,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.345211029052734,
      "rewards/margins": 1.401606559753418,
      "rewards/rejected": -44.7468147277832,
      "step": 3600
    },
    {
      "epoch": 1.926743602609132,
      "eval_logits/chosen": 7.481541156768799,
      "eval_logits/rejected": 7.208014965057373,
      "eval_logps/chosen": -42.3165168762207,
      "eval_logps/rejected": -43.4863395690918,
      "eval_loss": 2.4548253918510454e-07,
      "eval_rewards/accuracies": 0.5667656064033508,
      "eval_rewards/chosen": -42.3165168762207,
      "eval_rewards/margins": 1.169822096824646,
      "eval_rewards/rejected": -43.4863395690918,
      "eval_runtime": 40.3739,
      "eval_samples_per_second": 33.314,
      "eval_steps_per_second": 8.347,
      "step": 3600
    },
    {
      "epoch": 1.9294196353905335,
      "grad_norm": 1.0193225510697874e-07,
      "learning_rate": 3.401100613330605e-07,
      "logits/chosen": 9.300555229187012,
      "logits/rejected": 9.18540096282959,
      "logps/chosen": -43.99010467529297,
      "logps/rejected": -44.124046325683594,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.99010467529297,
      "rewards/margins": 0.13394129276275635,
      "rewards/rejected": -44.124046325683594,
      "step": 3605
    },
    {
      "epoch": 1.932095668171935,
      "grad_norm": 2.6554967928936995e-07,
      "learning_rate": 3.3863521299447514e-07,
      "logits/chosen": 9.391404151916504,
      "logits/rejected": 8.67957878112793,
      "logps/chosen": -42.815818786621094,
      "logps/rejected": -44.18699264526367,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -42.815818786621094,
      "rewards/margins": 1.371173620223999,
      "rewards/rejected": -44.18699264526367,
      "step": 3610
    },
    {
      "epoch": 1.9347717009533367,
      "grad_norm": 1.4645055223839736e-06,
      "learning_rate": 3.371619302177609e-07,
      "logits/chosen": 8.836959838867188,
      "logits/rejected": 8.367228507995605,
      "logps/chosen": -41.58897018432617,
      "logps/rejected": -43.02305603027344,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -41.58897018432617,
      "rewards/margins": 1.4340813159942627,
      "rewards/rejected": -43.02305603027344,
      "step": 3615
    },
    {
      "epoch": 1.9374477337347382,
      "grad_norm": 5.102221872778029e-08,
      "learning_rate": 3.3569022729671393e-07,
      "logits/chosen": 8.9774751663208,
      "logits/rejected": 8.841785430908203,
      "logps/chosen": -41.4398307800293,
      "logps/rejected": -42.85535430908203,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.4398307800293,
      "rewards/margins": 1.4155213832855225,
      "rewards/rejected": -42.85535430908203,
      "step": 3620
    },
    {
      "epoch": 1.9401237665161397,
      "grad_norm": 4.953960942036806e-08,
      "learning_rate": 3.342201185098024e-07,
      "logits/chosen": 8.801599502563477,
      "logits/rejected": 9.148088455200195,
      "logps/chosen": -42.89041519165039,
      "logps/rejected": -43.07294464111328,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.89041519165039,
      "rewards/margins": 0.18253205716609955,
      "rewards/rejected": -43.07294464111328,
      "step": 3625
    },
    {
      "epoch": 1.9427997992975414,
      "grad_norm": 4.628925314304078e-08,
      "learning_rate": 3.3275161812002807e-07,
      "logits/chosen": 8.998138427734375,
      "logits/rejected": 8.883317947387695,
      "logps/chosen": -42.71221160888672,
      "logps/rejected": -43.65961456298828,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.71221160888672,
      "rewards/margins": 0.9474003911018372,
      "rewards/rejected": -43.65961456298828,
      "step": 3630
    },
    {
      "epoch": 1.945475832078943,
      "grad_norm": 0.00021147751416318505,
      "learning_rate": 3.312847403747883e-07,
      "logits/chosen": 9.003536224365234,
      "logits/rejected": 8.807095527648926,
      "logps/chosen": -41.30414962768555,
      "logps/rejected": -42.76262283325195,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.30414962768555,
      "rewards/margins": 1.4584693908691406,
      "rewards/rejected": -42.76262283325195,
      "step": 3635
    },
    {
      "epoch": 1.9481518648603444,
      "grad_norm": 1.0298351919046356e-10,
      "learning_rate": 3.2981949950573733e-07,
      "logits/chosen": 8.763233184814453,
      "logits/rejected": 8.888627052307129,
      "logps/chosen": -42.269569396972656,
      "logps/rejected": -43.727325439453125,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.269569396972656,
      "rewards/margins": 1.4577535390853882,
      "rewards/rejected": -43.727325439453125,
      "step": 3640
    },
    {
      "epoch": 1.9508278976417461,
      "grad_norm": 2.4858901545262147e-10,
      "learning_rate": 3.283559097286486e-07,
      "logits/chosen": 8.83544921875,
      "logits/rejected": 8.6160888671875,
      "logps/chosen": -42.627220153808594,
      "logps/rejected": -44.04213333129883,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.627220153808594,
      "rewards/margins": 1.4149081707000732,
      "rewards/rejected": -44.04213333129883,
      "step": 3645
    },
    {
      "epoch": 1.9535039304231478,
      "grad_norm": 2.92707034353838e-12,
      "learning_rate": 3.268939852432765e-07,
      "logits/chosen": 8.995279312133789,
      "logits/rejected": 8.886581420898438,
      "logps/chosen": -42.7044677734375,
      "logps/rejected": -44.215675354003906,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.7044677734375,
      "rewards/margins": 1.511206865310669,
      "rewards/rejected": -44.215675354003906,
      "step": 3650
    },
    {
      "epoch": 1.9561799632045491,
      "grad_norm": 0.010264740524516293,
      "learning_rate": 3.254337402332187e-07,
      "logits/chosen": 9.067689895629883,
      "logits/rejected": 8.6453857421875,
      "logps/chosen": -43.12929916381836,
      "logps/rejected": -43.67707443237305,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.12929916381836,
      "rewards/margins": 0.5477749705314636,
      "rewards/rejected": -43.67707443237305,
      "step": 3655
    },
    {
      "epoch": 1.9588559959859508,
      "grad_norm": 3.314309570305559e-09,
      "learning_rate": 3.239751888657788e-07,
      "logits/chosen": 8.858835220336914,
      "logits/rejected": 8.436338424682617,
      "logps/chosen": -41.64826202392578,
      "logps/rejected": -43.458641052246094,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -41.64826202392578,
      "rewards/margins": 1.8103832006454468,
      "rewards/rejected": -43.458641052246094,
      "step": 3660
    },
    {
      "epoch": 1.9615320287673526,
      "grad_norm": 1.4664841053984346e-10,
      "learning_rate": 3.2251834529182856e-07,
      "logits/chosen": 9.15099048614502,
      "logits/rejected": 8.874698638916016,
      "logps/chosen": -43.49369430541992,
      "logps/rejected": -45.0528678894043,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.49369430541992,
      "rewards/margins": 1.559175968170166,
      "rewards/rejected": -45.0528678894043,
      "step": 3665
    },
    {
      "epoch": 1.9642080615487538,
      "grad_norm": 4.8363268309875536e-05,
      "learning_rate": 3.2106322364567075e-07,
      "logits/chosen": 9.21888542175293,
      "logits/rejected": 8.769892692565918,
      "logps/chosen": -41.953548431396484,
      "logps/rejected": -43.543636322021484,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -41.953548431396484,
      "rewards/margins": 1.5900886058807373,
      "rewards/rejected": -43.543636322021484,
      "step": 3670
    },
    {
      "epoch": 1.9668840943301555,
      "grad_norm": 1.3062468486591577e-10,
      "learning_rate": 3.1960983804490183e-07,
      "logits/chosen": 9.015542984008789,
      "logits/rejected": 8.798874855041504,
      "logps/chosen": -42.58515548706055,
      "logps/rejected": -44.5765380859375,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.58515548706055,
      "rewards/margins": 1.9913829565048218,
      "rewards/rejected": -44.5765380859375,
      "step": 3675
    },
    {
      "epoch": 1.9695601271115573,
      "grad_norm": 2.093880227717721e-08,
      "learning_rate": 3.1815820259027537e-07,
      "logits/chosen": 9.040103912353516,
      "logits/rejected": 8.717150688171387,
      "logps/chosen": -43.57097625732422,
      "logps/rejected": -44.8014030456543,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.57097625732422,
      "rewards/margins": 1.2304190397262573,
      "rewards/rejected": -44.8014030456543,
      "step": 3680
    },
    {
      "epoch": 1.9722361598929585,
      "grad_norm": 1.86839826568376e-09,
      "learning_rate": 3.16708331365565e-07,
      "logits/chosen": 8.841609954833984,
      "logits/rejected": 8.674072265625,
      "logps/chosen": -42.47706985473633,
      "logps/rejected": -43.49345016479492,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.47706985473633,
      "rewards/margins": 1.0163809061050415,
      "rewards/rejected": -43.49345016479492,
      "step": 3685
    },
    {
      "epoch": 1.9749121926743602,
      "grad_norm": 8.609434840677341e-05,
      "learning_rate": 3.152602384374275e-07,
      "logits/chosen": 9.062284469604492,
      "logits/rejected": 8.479073524475098,
      "logps/chosen": -41.911842346191406,
      "logps/rejected": -43.35440444946289,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -41.911842346191406,
      "rewards/margins": 1.4425618648529053,
      "rewards/rejected": -43.35440444946289,
      "step": 3690
    },
    {
      "epoch": 1.977588225455762,
      "grad_norm": 1.2428544009127452e-13,
      "learning_rate": 3.1381393785526697e-07,
      "logits/chosen": 8.972543716430664,
      "logits/rejected": 8.768500328063965,
      "logps/chosen": -43.57020568847656,
      "logps/rejected": -44.500789642333984,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -43.57020568847656,
      "rewards/margins": 0.9305828213691711,
      "rewards/rejected": -44.500789642333984,
      "step": 3695
    },
    {
      "epoch": 1.9802642582371635,
      "grad_norm": 0.000251858403495644,
      "learning_rate": 3.123694436510979e-07,
      "logits/chosen": 9.001150131225586,
      "logits/rejected": 8.65253734588623,
      "logps/chosen": -43.221378326416016,
      "logps/rejected": -44.03514862060547,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -43.221378326416016,
      "rewards/margins": 0.8137704730033875,
      "rewards/rejected": -44.03514862060547,
      "step": 3700
    },
    {
      "epoch": 1.982940291018565,
      "grad_norm": 1.1841891933569207e-12,
      "learning_rate": 3.1092676983940946e-07,
      "logits/chosen": 9.339698791503906,
      "logits/rejected": 8.944598197937012,
      "logps/chosen": -43.04975128173828,
      "logps/rejected": -44.066368103027344,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.04975128173828,
      "rewards/margins": 1.0166168212890625,
      "rewards/rejected": -44.066368103027344,
      "step": 3705
    },
    {
      "epoch": 1.9856163237999667,
      "grad_norm": 3.79940201829462e-06,
      "learning_rate": 3.094859304170293e-07,
      "logits/chosen": 8.634775161743164,
      "logits/rejected": 8.508970260620117,
      "logps/chosen": -41.86668395996094,
      "logps/rejected": -42.9535026550293,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.86668395996094,
      "rewards/margins": 1.0868192911148071,
      "rewards/rejected": -42.9535026550293,
      "step": 3710
    },
    {
      "epoch": 1.9882923565813682,
      "grad_norm": 2.4962069426964036e-11,
      "learning_rate": 3.0804693936298795e-07,
      "logits/chosen": 9.16926097869873,
      "logits/rejected": 9.15269660949707,
      "logps/chosen": -43.54453659057617,
      "logps/rejected": -44.353599548339844,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.54453659057617,
      "rewards/margins": 0.8090649843215942,
      "rewards/rejected": -44.353599548339844,
      "step": 3715
    },
    {
      "epoch": 1.9909683893627697,
      "grad_norm": 2.6387255405814323e-09,
      "learning_rate": 3.066098106383826e-07,
      "logits/chosen": 9.042624473571777,
      "logits/rejected": 8.616301536560059,
      "logps/chosen": -42.257225036621094,
      "logps/rejected": -42.584930419921875,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -42.257225036621094,
      "rewards/margins": 0.3277064263820648,
      "rewards/rejected": -42.584930419921875,
      "step": 3720
    },
    {
      "epoch": 1.9936444221441714,
      "grad_norm": 4.981580155066613e-09,
      "learning_rate": 3.0517455818624263e-07,
      "logits/chosen": 9.090360641479492,
      "logits/rejected": 8.818116188049316,
      "logps/chosen": -43.08317184448242,
      "logps/rejected": -43.85124588012695,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.08317184448242,
      "rewards/margins": 0.768067479133606,
      "rewards/rejected": -43.85124588012695,
      "step": 3725
    },
    {
      "epoch": 1.9963204549255729,
      "grad_norm": 2.6685157142407223e-08,
      "learning_rate": 3.037411959313936e-07,
      "logits/chosen": 9.131718635559082,
      "logits/rejected": 8.754563331604004,
      "logps/chosen": -41.910179138183594,
      "logps/rejected": -44.03877639770508,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -41.910179138183594,
      "rewards/margins": 2.128592014312744,
      "rewards/rejected": -44.03877639770508,
      "step": 3730
    },
    {
      "epoch": 1.9989964877069744,
      "grad_norm": 1.913456172737053e-08,
      "learning_rate": 3.023097377803224e-07,
      "logits/chosen": 8.980962753295898,
      "logits/rejected": 8.703187942504883,
      "logps/chosen": -43.56035614013672,
      "logps/rejected": -43.69626235961914,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -43.56035614013672,
      "rewards/margins": 0.13590554893016815,
      "rewards/rejected": -43.69626235961914,
      "step": 3735
    },
    {
      "epoch": 2.001672520488376,
      "grad_norm": 2.4971899323164673e-10,
      "learning_rate": 3.008801976210423e-07,
      "logits/chosen": 9.113046646118164,
      "logits/rejected": 8.793360710144043,
      "logps/chosen": -42.32197952270508,
      "logps/rejected": -41.993621826171875,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -42.32197952270508,
      "rewards/margins": -0.3283580541610718,
      "rewards/rejected": -41.993621826171875,
      "step": 3740
    },
    {
      "epoch": 2.0043485532697773,
      "grad_norm": 3.039977757036747e-11,
      "learning_rate": 2.994525893229581e-07,
      "logits/chosen": 9.052238464355469,
      "logits/rejected": 8.735589981079102,
      "logps/chosen": -43.29452133178711,
      "logps/rejected": -45.156394958496094,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.29452133178711,
      "rewards/margins": 1.8618745803833008,
      "rewards/rejected": -45.156394958496094,
      "step": 3745
    },
    {
      "epoch": 2.007024586051179,
      "grad_norm": 1.1388265880232565e-06,
      "learning_rate": 2.98026926736732e-07,
      "logits/chosen": 9.030289649963379,
      "logits/rejected": 8.800799369812012,
      "logps/chosen": -42.60272979736328,
      "logps/rejected": -43.9164924621582,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.60272979736328,
      "rewards/margins": 1.3137590885162354,
      "rewards/rejected": -43.9164924621582,
      "step": 3750
    },
    {
      "epoch": 2.0097006188325808,
      "grad_norm": 1.817417636378003e-11,
      "learning_rate": 2.9660322369414846e-07,
      "logits/chosen": 8.969718933105469,
      "logits/rejected": 8.599370956420898,
      "logps/chosen": -41.9445686340332,
      "logps/rejected": -43.792171478271484,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -41.9445686340332,
      "rewards/margins": 1.847603440284729,
      "rewards/rejected": -43.792171478271484,
      "step": 3755
    },
    {
      "epoch": 2.0123766516139825,
      "grad_norm": 4.82318020708712e-10,
      "learning_rate": 2.9518149400798063e-07,
      "logits/chosen": 9.258691787719727,
      "logits/rejected": 9.018590927124023,
      "logps/chosen": -41.618587493896484,
      "logps/rejected": -42.92237091064453,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.618587493896484,
      "rewards/margins": 1.3037803173065186,
      "rewards/rejected": -42.92237091064453,
      "step": 3760
    },
    {
      "epoch": 2.0150526843953838,
      "grad_norm": 3.2625408529560704e-10,
      "learning_rate": 2.9376175147185633e-07,
      "logits/chosen": 9.354743003845215,
      "logits/rejected": 8.688486099243164,
      "logps/chosen": -43.16361618041992,
      "logps/rejected": -44.503639221191406,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.16361618041992,
      "rewards/margins": 1.3400237560272217,
      "rewards/rejected": -44.503639221191406,
      "step": 3765
    },
    {
      "epoch": 2.0177287171767855,
      "grad_norm": 1.1563170913060482e-13,
      "learning_rate": 2.9234400986012376e-07,
      "logits/chosen": 9.151975631713867,
      "logits/rejected": 8.601824760437012,
      "logps/chosen": -42.729515075683594,
      "logps/rejected": -44.817352294921875,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -42.729515075683594,
      "rewards/margins": 2.0878400802612305,
      "rewards/rejected": -44.817352294921875,
      "step": 3770
    },
    {
      "epoch": 2.020404749958187,
      "grad_norm": 8.980844137640453e-10,
      "learning_rate": 2.9092828292771817e-07,
      "logits/chosen": 9.241704940795898,
      "logits/rejected": 9.087262153625488,
      "logps/chosen": -43.182579040527344,
      "logps/rejected": -44.141822814941406,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.182579040527344,
      "rewards/margins": 0.9592512845993042,
      "rewards/rejected": -44.141822814941406,
      "step": 3775
    },
    {
      "epoch": 2.0230807827395885,
      "grad_norm": 7.543454577515874e-10,
      "learning_rate": 2.8951458441002875e-07,
      "logits/chosen": 8.907041549682617,
      "logits/rejected": 8.689894676208496,
      "logps/chosen": -41.20647430419922,
      "logps/rejected": -41.67596435546875,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -41.20647430419922,
      "rewards/margins": 0.4694964289665222,
      "rewards/rejected": -41.67596435546875,
      "step": 3780
    },
    {
      "epoch": 2.02575681552099,
      "grad_norm": 1.4743813651592606e-07,
      "learning_rate": 2.881029280227643e-07,
      "logits/chosen": 9.202451705932617,
      "logits/rejected": 8.600340843200684,
      "logps/chosen": -42.56684875488281,
      "logps/rejected": -43.63859939575195,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -42.56684875488281,
      "rewards/margins": 1.0717495679855347,
      "rewards/rejected": -43.63859939575195,
      "step": 3785
    },
    {
      "epoch": 2.028432848302392,
      "grad_norm": 3.0150184532156505e-11,
      "learning_rate": 2.8669332746182177e-07,
      "logits/chosen": 9.287517547607422,
      "logits/rejected": 8.754729270935059,
      "logps/chosen": -42.62308883666992,
      "logps/rejected": -44.30974578857422,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -42.62308883666992,
      "rewards/margins": 1.6866579055786133,
      "rewards/rejected": -44.30974578857422,
      "step": 3790
    },
    {
      "epoch": 2.031108881083793,
      "grad_norm": 1.0156104208237597e-05,
      "learning_rate": 2.8528579640315156e-07,
      "logits/chosen": 8.80391788482666,
      "logits/rejected": 8.872693061828613,
      "logps/chosen": -43.396141052246094,
      "logps/rejected": -43.928218841552734,
      "loss": 0.0003,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.396141052246094,
      "rewards/margins": 0.5320850610733032,
      "rewards/rejected": -43.928218841552734,
      "step": 3795
    },
    {
      "epoch": 2.033784913865195,
      "grad_norm": 6.576066294088537e-07,
      "learning_rate": 2.8388034850262646e-07,
      "logits/chosen": 8.98240852355957,
      "logits/rejected": 8.529772758483887,
      "logps/chosen": -43.47077560424805,
      "logps/rejected": -44.37076950073242,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.47077560424805,
      "rewards/margins": 0.8999907374382019,
      "rewards/rejected": -44.37076950073242,
      "step": 3800
    },
    {
      "epoch": 2.0364609466465966,
      "grad_norm": 8.468351144692882e-09,
      "learning_rate": 2.824769973959079e-07,
      "logits/chosen": 9.085939407348633,
      "logits/rejected": 8.778947830200195,
      "logps/chosen": -42.20588684082031,
      "logps/rejected": -44.11560821533203,
      "loss": 0.0,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -42.20588684082031,
      "rewards/margins": 1.9097156524658203,
      "rewards/rejected": -44.11560821533203,
      "step": 3805
    },
    {
      "epoch": 2.039136979427998,
      "grad_norm": 1.0614687667194108e-05,
      "learning_rate": 2.81075756698315e-07,
      "logits/chosen": 8.683633804321289,
      "logits/rejected": 8.692230224609375,
      "logps/chosen": -43.65705108642578,
      "logps/rejected": -45.27473449707031,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.65705108642578,
      "rewards/margins": 1.6176856756210327,
      "rewards/rejected": -45.27473449707031,
      "step": 3810
    },
    {
      "epoch": 2.0418130122093996,
      "grad_norm": 9.635794652609165e-13,
      "learning_rate": 2.7967664000469035e-07,
      "logits/chosen": 9.182882308959961,
      "logits/rejected": 8.858095169067383,
      "logps/chosen": -43.048606872558594,
      "logps/rejected": -44.22917938232422,
      "loss": 0.0005,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -43.048606872558594,
      "rewards/margins": 1.180572271347046,
      "rewards/rejected": -44.22917938232422,
      "step": 3815
    },
    {
      "epoch": 2.0444890449908013,
      "grad_norm": 4.295626604099632e-09,
      "learning_rate": 2.7827966088927095e-07,
      "logits/chosen": 9.295331001281738,
      "logits/rejected": 8.602898597717285,
      "logps/chosen": -43.672733306884766,
      "logps/rejected": -45.004112243652344,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.672733306884766,
      "rewards/margins": 1.3313796520233154,
      "rewards/rejected": -45.004112243652344,
      "step": 3820
    },
    {
      "epoch": 2.0471650777722026,
      "grad_norm": 1.3777344597859796e-10,
      "learning_rate": 2.768848329055538e-07,
      "logits/chosen": 9.324398040771484,
      "logits/rejected": 9.085660934448242,
      "logps/chosen": -43.610008239746094,
      "logps/rejected": -45.07090759277344,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -43.610008239746094,
      "rewards/margins": 1.46089768409729,
      "rewards/rejected": -45.07090759277344,
      "step": 3825
    },
    {
      "epoch": 2.0498411105536043,
      "grad_norm": 4.2595106314453214e-10,
      "learning_rate": 2.7549216958616657e-07,
      "logits/chosen": 9.324460983276367,
      "logits/rejected": 8.689237594604492,
      "logps/chosen": -42.19231414794922,
      "logps/rejected": -43.17029571533203,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.19231414794922,
      "rewards/margins": 0.9779860377311707,
      "rewards/rejected": -43.17029571533203,
      "step": 3830
    },
    {
      "epoch": 2.052517143335006,
      "grad_norm": 3.211331088623185e-06,
      "learning_rate": 2.741016844427344e-07,
      "logits/chosen": 8.99532413482666,
      "logits/rejected": 8.408125877380371,
      "logps/chosen": -42.37995529174805,
      "logps/rejected": -43.69165802001953,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.37995529174805,
      "rewards/margins": 1.3117038011550903,
      "rewards/rejected": -43.69165802001953,
      "step": 3835
    },
    {
      "epoch": 2.0551931761164073,
      "grad_norm": 8.06253437927408e-09,
      "learning_rate": 2.7271339096575073e-07,
      "logits/chosen": 9.014673233032227,
      "logits/rejected": 8.615967750549316,
      "logps/chosen": -42.85087585449219,
      "logps/rejected": -44.701324462890625,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -42.85087585449219,
      "rewards/margins": 1.8504464626312256,
      "rewards/rejected": -44.701324462890625,
      "step": 3840
    },
    {
      "epoch": 2.057869208897809,
      "grad_norm": 5.721844444063363e-13,
      "learning_rate": 2.713273026244446e-07,
      "logits/chosen": 9.721975326538086,
      "logits/rejected": 8.899737358093262,
      "logps/chosen": -42.51079559326172,
      "logps/rejected": -44.6930046081543,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -42.51079559326172,
      "rewards/margins": 2.182206869125366,
      "rewards/rejected": -44.6930046081543,
      "step": 3845
    },
    {
      "epoch": 2.0605452416792107,
      "grad_norm": 2.1517434797084815e-07,
      "learning_rate": 2.6994343286665156e-07,
      "logits/chosen": 9.298352241516113,
      "logits/rejected": 8.801363945007324,
      "logps/chosen": -41.07033157348633,
      "logps/rejected": -43.37969970703125,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -41.07033157348633,
      "rewards/margins": 2.3093700408935547,
      "rewards/rejected": -43.37969970703125,
      "step": 3850
    },
    {
      "epoch": 2.063221274460612,
      "grad_norm": 1.5295456626605476e-05,
      "learning_rate": 2.6856179511868156e-07,
      "logits/chosen": 9.238503456115723,
      "logits/rejected": 8.465252876281738,
      "logps/chosen": -41.18201446533203,
      "logps/rejected": -42.878196716308594,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.18201446533203,
      "rewards/margins": 1.6961853504180908,
      "rewards/rejected": -42.878196716308594,
      "step": 3855
    },
    {
      "epoch": 2.0658973072420137,
      "grad_norm": 6.60994330410182e-09,
      "learning_rate": 2.6718240278519056e-07,
      "logits/chosen": 8.798654556274414,
      "logits/rejected": 8.263616561889648,
      "logps/chosen": -41.15531539916992,
      "logps/rejected": -43.32644271850586,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -41.15531539916992,
      "rewards/margins": 2.171125888824463,
      "rewards/rejected": -43.32644271850586,
      "step": 3860
    },
    {
      "epoch": 2.0685733400234154,
      "grad_norm": 3.128752490463152e-10,
      "learning_rate": 2.6580526924904866e-07,
      "logits/chosen": 9.404783248901367,
      "logits/rejected": 8.739439964294434,
      "logps/chosen": -43.65899658203125,
      "logps/rejected": -45.323692321777344,
      "loss": 0.001,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -43.65899658203125,
      "rewards/margins": 1.6646976470947266,
      "rewards/rejected": -45.323692321777344,
      "step": 3865
    },
    {
      "epoch": 2.0712493728048167,
      "grad_norm": 6.715896231778555e-08,
      "learning_rate": 2.6443040787121186e-07,
      "logits/chosen": 9.327392578125,
      "logits/rejected": 8.963663101196289,
      "logps/chosen": -43.72232437133789,
      "logps/rejected": -44.819427490234375,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.72232437133789,
      "rewards/margins": 1.097109079360962,
      "rewards/rejected": -44.819427490234375,
      "step": 3870
    },
    {
      "epoch": 2.0739254055862184,
      "grad_norm": 2.004862467346365e-07,
      "learning_rate": 2.6305783199059084e-07,
      "logits/chosen": 8.901762962341309,
      "logits/rejected": 8.742029190063477,
      "logps/chosen": -42.300865173339844,
      "logps/rejected": -44.190460205078125,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.300865173339844,
      "rewards/margins": 1.8895998001098633,
      "rewards/rejected": -44.190460205078125,
      "step": 3875
    },
    {
      "epoch": 2.07660143836762,
      "grad_norm": 1.8011284678424387e-13,
      "learning_rate": 2.6168755492392324e-07,
      "logits/chosen": 9.4299955368042,
      "logits/rejected": 8.773306846618652,
      "logps/chosen": -42.846839904785156,
      "logps/rejected": -44.283424377441406,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.846839904785156,
      "rewards/margins": 1.436584234237671,
      "rewards/rejected": -44.283424377441406,
      "step": 3880
    },
    {
      "epoch": 2.0792774711490214,
      "grad_norm": 0.001467624750922948,
      "learning_rate": 2.6031958996564274e-07,
      "logits/chosen": 9.310066223144531,
      "logits/rejected": 8.857309341430664,
      "logps/chosen": -43.28919219970703,
      "logps/rejected": -44.32575988769531,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.28919219970703,
      "rewards/margins": 1.036568522453308,
      "rewards/rejected": -44.32575988769531,
      "step": 3885
    },
    {
      "epoch": 2.081953503930423,
      "grad_norm": 1.576593305171687e-09,
      "learning_rate": 2.589539503877518e-07,
      "logits/chosen": 8.669404983520508,
      "logits/rejected": 8.387873649597168,
      "logps/chosen": -43.117332458496094,
      "logps/rejected": -44.266868591308594,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.117332458496094,
      "rewards/margins": 1.1495267152786255,
      "rewards/rejected": -44.266868591308594,
      "step": 3890
    },
    {
      "epoch": 2.084629536711825,
      "grad_norm": 1.1491102099131159e-07,
      "learning_rate": 2.5759064943969125e-07,
      "logits/chosen": 9.045702934265137,
      "logits/rejected": 8.297749519348145,
      "logps/chosen": -42.43684005737305,
      "logps/rejected": -44.2757568359375,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.43684005737305,
      "rewards/margins": 1.8389114141464233,
      "rewards/rejected": -44.2757568359375,
      "step": 3895
    },
    {
      "epoch": 2.087305569493226,
      "grad_norm": 1.741351840835168e-08,
      "learning_rate": 2.562297003482131e-07,
      "logits/chosen": 9.002426147460938,
      "logits/rejected": 9.01283073425293,
      "logps/chosen": -43.30495071411133,
      "logps/rejected": -43.95606231689453,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -43.30495071411133,
      "rewards/margins": 0.6511114835739136,
      "rewards/rejected": -43.95606231689453,
      "step": 3900
    },
    {
      "epoch": 2.089981602274628,
      "grad_norm": 8.817827689403988e-09,
      "learning_rate": 2.548711163172512e-07,
      "logits/chosen": 9.014253616333008,
      "logits/rejected": 8.92760944366455,
      "logps/chosen": -42.65181350708008,
      "logps/rejected": -44.23887634277344,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.65181350708008,
      "rewards/margins": 1.5870609283447266,
      "rewards/rejected": -44.23887634277344,
      "step": 3905
    },
    {
      "epoch": 2.0926576350560295,
      "grad_norm": 4.1983228105477875e-10,
      "learning_rate": 2.53514910527794e-07,
      "logits/chosen": 9.045778274536133,
      "logits/rejected": 8.59282398223877,
      "logps/chosen": -43.62392044067383,
      "logps/rejected": -44.38840103149414,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.62392044067383,
      "rewards/margins": 0.7644834518432617,
      "rewards/rejected": -44.38840103149414,
      "step": 3910
    },
    {
      "epoch": 2.095333667837431,
      "grad_norm": 2.5817597533685205e-11,
      "learning_rate": 2.5216109613775573e-07,
      "logits/chosen": 9.067813873291016,
      "logits/rejected": 8.734966278076172,
      "logps/chosen": -43.35368728637695,
      "logps/rejected": -44.36429977416992,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.35368728637695,
      "rewards/margins": 1.0106127262115479,
      "rewards/rejected": -44.36429977416992,
      "step": 3915
    },
    {
      "epoch": 2.0980097006188325,
      "grad_norm": 7.103466358065859e-07,
      "learning_rate": 2.5080968628184993e-07,
      "logits/chosen": 8.974212646484375,
      "logits/rejected": 8.480744361877441,
      "logps/chosen": -42.90849685668945,
      "logps/rejected": -44.19004440307617,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.90849685668945,
      "rewards/margins": 1.281553864479065,
      "rewards/rejected": -44.19004440307617,
      "step": 3920
    },
    {
      "epoch": 2.1006857334002342,
      "grad_norm": 7.472472465856852e-10,
      "learning_rate": 2.494606940714605e-07,
      "logits/chosen": 8.834748268127441,
      "logits/rejected": 8.467195510864258,
      "logps/chosen": -42.65045928955078,
      "logps/rejected": -43.457786560058594,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.65045928955078,
      "rewards/margins": 0.8073281049728394,
      "rewards/rejected": -43.457786560058594,
      "step": 3925
    },
    {
      "epoch": 2.103361766181636,
      "grad_norm": 7.009659713125854e-08,
      "learning_rate": 2.4811413259451625e-07,
      "logits/chosen": 9.188947677612305,
      "logits/rejected": 8.615166664123535,
      "logps/chosen": -41.76762008666992,
      "logps/rejected": -43.74374771118164,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -41.76762008666992,
      "rewards/margins": 1.9761216640472412,
      "rewards/rejected": -43.74374771118164,
      "step": 3930
    },
    {
      "epoch": 2.106037798963037,
      "grad_norm": 3.2265033106407376e-10,
      "learning_rate": 2.46770014915362e-07,
      "logits/chosen": 9.099128723144531,
      "logits/rejected": 8.857921600341797,
      "logps/chosen": -43.18244934082031,
      "logps/rejected": -44.17724609375,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.18244934082031,
      "rewards/margins": 0.9947935342788696,
      "rewards/rejected": -44.17724609375,
      "step": 3935
    },
    {
      "epoch": 2.108713831744439,
      "grad_norm": 3.831122985380686e-12,
      "learning_rate": 2.45428354074634e-07,
      "logits/chosen": 8.956808090209961,
      "logits/rejected": 8.80980396270752,
      "logps/chosen": -43.34827423095703,
      "logps/rejected": -45.02986526489258,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.34827423095703,
      "rewards/margins": 1.6815884113311768,
      "rewards/rejected": -45.02986526489258,
      "step": 3940
    },
    {
      "epoch": 2.1113898645258407,
      "grad_norm": 5.385973620598079e-09,
      "learning_rate": 2.4408916308913105e-07,
      "logits/chosen": 8.856588363647461,
      "logits/rejected": 8.372272491455078,
      "logps/chosen": -41.744720458984375,
      "logps/rejected": -44.288475036621094,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -41.744720458984375,
      "rewards/margins": 2.543750286102295,
      "rewards/rejected": -44.288475036621094,
      "step": 3945
    },
    {
      "epoch": 2.114065897307242,
      "grad_norm": 4.2885550934086666e-11,
      "learning_rate": 2.4275245495169025e-07,
      "logits/chosen": 9.060127258300781,
      "logits/rejected": 8.5348539352417,
      "logps/chosen": -43.25366973876953,
      "logps/rejected": -44.860923767089844,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.25366973876953,
      "rewards/margins": 1.6072571277618408,
      "rewards/rejected": -44.860923767089844,
      "step": 3950
    },
    {
      "epoch": 2.1167419300886436,
      "grad_norm": 1.0220418821000886e-08,
      "learning_rate": 2.414182426310597e-07,
      "logits/chosen": 9.218419075012207,
      "logits/rejected": 9.203883171081543,
      "logps/chosen": -43.59880828857422,
      "logps/rejected": -44.50844955444336,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.59880828857422,
      "rewards/margins": 0.9096387624740601,
      "rewards/rejected": -44.50844955444336,
      "step": 3955
    },
    {
      "epoch": 2.1194179628700454,
      "grad_norm": 4.893393021176766e-08,
      "learning_rate": 2.400865390717734e-07,
      "logits/chosen": 9.099970817565918,
      "logits/rejected": 8.677289009094238,
      "logps/chosen": -42.886451721191406,
      "logps/rejected": -44.28646469116211,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.886451721191406,
      "rewards/margins": 1.4000129699707031,
      "rewards/rejected": -44.28646469116211,
      "step": 3960
    },
    {
      "epoch": 2.1220939956514466,
      "grad_norm": 7.48536826683026e-07,
      "learning_rate": 2.3875735719402475e-07,
      "logits/chosen": 9.244757652282715,
      "logits/rejected": 8.968472480773926,
      "logps/chosen": -43.204368591308594,
      "logps/rejected": -44.411964416503906,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.204368591308594,
      "rewards/margins": 1.2075891494750977,
      "rewards/rejected": -44.411964416503906,
      "step": 3965
    },
    {
      "epoch": 2.1247700284328483,
      "grad_norm": 4.523063610498576e-10,
      "learning_rate": 2.3743070989354258e-07,
      "logits/chosen": 9.296489715576172,
      "logits/rejected": 9.018621444702148,
      "logps/chosen": -43.32949447631836,
      "logps/rejected": -45.09051513671875,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.32949447631836,
      "rewards/margins": 1.7610194683074951,
      "rewards/rejected": -45.09051513671875,
      "step": 3970
    },
    {
      "epoch": 2.12744606121425,
      "grad_norm": 2.3776824452666e-09,
      "learning_rate": 2.3610661004146454e-07,
      "logits/chosen": 9.050966262817383,
      "logits/rejected": 8.661293029785156,
      "logps/chosen": -42.240291595458984,
      "logps/rejected": -43.407020568847656,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.240291595458984,
      "rewards/margins": 1.1667273044586182,
      "rewards/rejected": -43.407020568847656,
      "step": 3975
    },
    {
      "epoch": 2.1301220939956513,
      "grad_norm": 2.625804320271655e-05,
      "learning_rate": 2.3478507048421314e-07,
      "logits/chosen": 9.038867950439453,
      "logits/rejected": 8.616304397583008,
      "logps/chosen": -42.380035400390625,
      "logps/rejected": -43.47580337524414,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -42.380035400390625,
      "rewards/margins": 1.0957672595977783,
      "rewards/rejected": -43.47580337524414,
      "step": 3980
    },
    {
      "epoch": 2.132798126777053,
      "grad_norm": 1.9259894518899858e-12,
      "learning_rate": 2.334661040433713e-07,
      "logits/chosen": 9.501883506774902,
      "logits/rejected": 9.05534553527832,
      "logps/chosen": -44.69688415527344,
      "logps/rejected": -46.294898986816406,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.69688415527344,
      "rewards/margins": 1.5980141162872314,
      "rewards/rejected": -46.294898986816406,
      "step": 3985
    },
    {
      "epoch": 2.1354741595584548,
      "grad_norm": 3.2922434755463565e-10,
      "learning_rate": 2.321497235155568e-07,
      "logits/chosen": 9.28370475769043,
      "logits/rejected": 8.744659423828125,
      "logps/chosen": -42.807228088378906,
      "logps/rejected": -44.01629638671875,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.807228088378906,
      "rewards/margins": 1.2090747356414795,
      "rewards/rejected": -44.01629638671875,
      "step": 3990
    },
    {
      "epoch": 2.138150192339856,
      "grad_norm": 2.4786456834887667e-10,
      "learning_rate": 2.3083594167229965e-07,
      "logits/chosen": 9.276188850402832,
      "logits/rejected": 8.40411376953125,
      "logps/chosen": -42.7730598449707,
      "logps/rejected": -44.181884765625,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.7730598449707,
      "rewards/margins": 1.4088283777236938,
      "rewards/rejected": -44.181884765625,
      "step": 3995
    },
    {
      "epoch": 2.1408262251212578,
      "grad_norm": 1.3875723591921611e-08,
      "learning_rate": 2.295247712599167e-07,
      "logits/chosen": 9.039514541625977,
      "logits/rejected": 8.849933624267578,
      "logps/chosen": -43.12586212158203,
      "logps/rejected": -44.19435501098633,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.12586212158203,
      "rewards/margins": 1.0684914588928223,
      "rewards/rejected": -44.19435501098633,
      "step": 4000
    },
    {
      "epoch": 2.1408262251212578,
      "eval_logits/chosen": 7.554755687713623,
      "eval_logits/rejected": 7.255157947540283,
      "eval_logps/chosen": -43.03852081298828,
      "eval_logps/rejected": -44.147335052490234,
      "eval_loss": 2.646533232564252e-07,
      "eval_rewards/accuracies": 0.5697329640388489,
      "eval_rewards/chosen": -43.03852081298828,
      "eval_rewards/margins": 1.1088168621063232,
      "eval_rewards/rejected": -44.147335052490234,
      "eval_runtime": 40.3659,
      "eval_samples_per_second": 33.32,
      "eval_steps_per_second": 8.349,
      "step": 4000
    },
    {
      "epoch": 2.1435022579026595,
      "grad_norm": 8.014969882167829e-06,
      "learning_rate": 2.2821622499938948e-07,
      "logits/chosen": 8.994928359985352,
      "logits/rejected": 8.230422973632812,
      "logps/chosen": -42.28566360473633,
      "logps/rejected": -43.514556884765625,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.28566360473633,
      "rewards/margins": 1.228898286819458,
      "rewards/rejected": -43.514556884765625,
      "step": 4005
    },
    {
      "epoch": 2.1461782906840607,
      "grad_norm": 3.517083628322829e-12,
      "learning_rate": 2.269103155862391e-07,
      "logits/chosen": 9.259135246276855,
      "logits/rejected": 8.679240226745605,
      "logps/chosen": -43.896732330322266,
      "logps/rejected": -44.3692512512207,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.896732330322266,
      "rewards/margins": 0.47251325845718384,
      "rewards/rejected": -44.3692512512207,
      "step": 4010
    },
    {
      "epoch": 2.1488543234654625,
      "grad_norm": 1.3701373141855658e-07,
      "learning_rate": 2.2560705569040483e-07,
      "logits/chosen": 9.022557258605957,
      "logits/rejected": 8.053827285766602,
      "logps/chosen": -43.2159309387207,
      "logps/rejected": -44.100799560546875,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.2159309387207,
      "rewards/margins": 0.8848685026168823,
      "rewards/rejected": -44.100799560546875,
      "step": 4015
    },
    {
      "epoch": 2.151530356246864,
      "grad_norm": 6.404427860088793e-07,
      "learning_rate": 2.2430645795611963e-07,
      "logits/chosen": 9.072053909301758,
      "logits/rejected": 8.639387130737305,
      "logps/chosen": -42.088523864746094,
      "logps/rejected": -43.122013092041016,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.088523864746094,
      "rewards/margins": 1.0334886312484741,
      "rewards/rejected": -43.122013092041016,
      "step": 4020
    },
    {
      "epoch": 2.1542063890282654,
      "grad_norm": 1.186050331956477e-06,
      "learning_rate": 2.230085350017884e-07,
      "logits/chosen": 8.985651016235352,
      "logits/rejected": 8.547399520874023,
      "logps/chosen": -42.805233001708984,
      "logps/rejected": -43.92900848388672,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -42.805233001708984,
      "rewards/margins": 1.1237679719924927,
      "rewards/rejected": -43.92900848388672,
      "step": 4025
    },
    {
      "epoch": 2.156882421809667,
      "grad_norm": 5.068366124293054e-14,
      "learning_rate": 2.2171329941986554e-07,
      "logits/chosen": 9.14936637878418,
      "logits/rejected": 8.997502326965332,
      "logps/chosen": -43.808631896972656,
      "logps/rejected": -45.30188751220703,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.808631896972656,
      "rewards/margins": 1.493253231048584,
      "rewards/rejected": -45.30188751220703,
      "step": 4030
    },
    {
      "epoch": 2.159558454591069,
      "grad_norm": 2.1810545762746e-10,
      "learning_rate": 2.2042076377673202e-07,
      "logits/chosen": 8.72819709777832,
      "logits/rejected": 8.76697826385498,
      "logps/chosen": -42.61308670043945,
      "logps/rejected": -43.672000885009766,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.61308670043945,
      "rewards/margins": 1.0589206218719482,
      "rewards/rejected": -43.672000885009766,
      "step": 4035
    },
    {
      "epoch": 2.16223448737247,
      "grad_norm": 1.2177576135311725e-09,
      "learning_rate": 2.1913094061257476e-07,
      "logits/chosen": 8.976733207702637,
      "logits/rejected": 8.85966682434082,
      "logps/chosen": -43.48511505126953,
      "logps/rejected": -44.28733444213867,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.48511505126953,
      "rewards/margins": 0.8022161722183228,
      "rewards/rejected": -44.28733444213867,
      "step": 4040
    },
    {
      "epoch": 2.164910520153872,
      "grad_norm": 1.733320928040242e-10,
      "learning_rate": 2.178438424412633e-07,
      "logits/chosen": 8.916051864624023,
      "logits/rejected": 8.544427871704102,
      "logps/chosen": -43.135459899902344,
      "logps/rejected": -44.476280212402344,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.135459899902344,
      "rewards/margins": 1.3408172130584717,
      "rewards/rejected": -44.476280212402344,
      "step": 4045
    },
    {
      "epoch": 2.1675865529352736,
      "grad_norm": 6.924030041514536e-09,
      "learning_rate": 2.165594817502302e-07,
      "logits/chosen": 9.367121696472168,
      "logits/rejected": 8.813446044921875,
      "logps/chosen": -43.04452896118164,
      "logps/rejected": -44.201663970947266,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.04452896118164,
      "rewards/margins": 1.157134771347046,
      "rewards/rejected": -44.201663970947266,
      "step": 4050
    },
    {
      "epoch": 2.170262585716675,
      "grad_norm": 5.88410062128727e-09,
      "learning_rate": 2.1527787100034806e-07,
      "logits/chosen": 8.773908615112305,
      "logits/rejected": 8.69034194946289,
      "logps/chosen": -42.485679626464844,
      "logps/rejected": -43.780277252197266,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.485679626464844,
      "rewards/margins": 1.2945966720581055,
      "rewards/rejected": -43.780277252197266,
      "step": 4055
    },
    {
      "epoch": 2.1729386184980766,
      "grad_norm": 7.435395033790735e-08,
      "learning_rate": 2.1399902262581037e-07,
      "logits/chosen": 9.17042064666748,
      "logits/rejected": 8.571090698242188,
      "logps/chosen": -42.98744583129883,
      "logps/rejected": -44.34369659423828,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -42.98744583129883,
      "rewards/margins": 1.3562500476837158,
      "rewards/rejected": -44.34369659423828,
      "step": 4060
    },
    {
      "epoch": 2.1756146512794783,
      "grad_norm": 3.1036007844510176e-10,
      "learning_rate": 2.127229490340094e-07,
      "logits/chosen": 9.034303665161133,
      "logits/rejected": 8.611432075500488,
      "logps/chosen": -43.0045166015625,
      "logps/rejected": -44.0695686340332,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.0045166015625,
      "rewards/margins": 1.0650520324707031,
      "rewards/rejected": -44.0695686340332,
      "step": 4065
    },
    {
      "epoch": 2.1782906840608796,
      "grad_norm": 2.2912627928083715e-08,
      "learning_rate": 2.1144966260541698e-07,
      "logits/chosen": 9.093645095825195,
      "logits/rejected": 8.371310234069824,
      "logps/chosen": -42.680213928222656,
      "logps/rejected": -43.944541931152344,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.680213928222656,
      "rewards/margins": 1.264331579208374,
      "rewards/rejected": -43.944541931152344,
      "step": 4070
    },
    {
      "epoch": 2.1809667168422813,
      "grad_norm": 6.547996878046198e-05,
      "learning_rate": 2.1017917569346332e-07,
      "logits/chosen": 9.030420303344727,
      "logits/rejected": 8.363921165466309,
      "logps/chosen": -42.09078598022461,
      "logps/rejected": -43.690826416015625,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.09078598022461,
      "rewards/margins": 1.6000381708145142,
      "rewards/rejected": -43.690826416015625,
      "step": 4075
    },
    {
      "epoch": 2.183642749623683,
      "grad_norm": 0.00018978273997853653,
      "learning_rate": 2.0891150062441837e-07,
      "logits/chosen": 9.048044204711914,
      "logits/rejected": 8.46544075012207,
      "logps/chosen": -42.538658142089844,
      "logps/rejected": -44.154579162597656,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.538658142089844,
      "rewards/margins": 1.615917444229126,
      "rewards/rejected": -44.154579162597656,
      "step": 4080
    },
    {
      "epoch": 2.1863187824050843,
      "grad_norm": 7.93023309397749e-12,
      "learning_rate": 2.0764664969727086e-07,
      "logits/chosen": 8.71142864227295,
      "logits/rejected": 8.834856986999512,
      "logps/chosen": -41.934146881103516,
      "logps/rejected": -42.829708099365234,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -41.934146881103516,
      "rewards/margins": 0.8955645561218262,
      "rewards/rejected": -42.829708099365234,
      "step": 4085
    },
    {
      "epoch": 2.188994815186486,
      "grad_norm": 1.562234346281564e-09,
      "learning_rate": 2.0638463518361033e-07,
      "logits/chosen": 9.596474647521973,
      "logits/rejected": 8.892104148864746,
      "logps/chosen": -42.5695915222168,
      "logps/rejected": -43.9444465637207,
      "loss": 0.0001,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.5695915222168,
      "rewards/margins": 1.3748515844345093,
      "rewards/rejected": -43.9444465637207,
      "step": 4090
    },
    {
      "epoch": 2.1916708479678877,
      "grad_norm": 6.821408193988347e-11,
      "learning_rate": 2.0512546932750702e-07,
      "logits/chosen": 9.35791015625,
      "logits/rejected": 9.129060745239258,
      "logps/chosen": -42.6036376953125,
      "logps/rejected": -43.65306854248047,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -42.6036376953125,
      "rewards/margins": 1.0494320392608643,
      "rewards/rejected": -43.65306854248047,
      "step": 4095
    },
    {
      "epoch": 2.194346880749289,
      "grad_norm": 1.196708590046929e-06,
      "learning_rate": 2.0386916434539343e-07,
      "logits/chosen": 9.000151634216309,
      "logits/rejected": 8.360278129577637,
      "logps/chosen": -44.51485061645508,
      "logps/rejected": -45.5093994140625,
      "loss": 0.0005,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.51485061645508,
      "rewards/margins": 0.9945542216300964,
      "rewards/rejected": -45.5093994140625,
      "step": 4100
    },
    {
      "epoch": 2.1970229135306907,
      "grad_norm": 4.4444548241558217e-10,
      "learning_rate": 2.0261573242594627e-07,
      "logits/chosen": 9.081033706665039,
      "logits/rejected": 8.563546180725098,
      "logps/chosen": -42.82344055175781,
      "logps/rejected": -43.670806884765625,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.82344055175781,
      "rewards/margins": 0.8473671078681946,
      "rewards/rejected": -43.670806884765625,
      "step": 4105
    },
    {
      "epoch": 2.1996989463120924,
      "grad_norm": 3.007488682186981e-11,
      "learning_rate": 2.0136518572996724e-07,
      "logits/chosen": 9.229379653930664,
      "logits/rejected": 8.714961051940918,
      "logps/chosen": -43.3348274230957,
      "logps/rejected": -44.753509521484375,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.3348274230957,
      "rewards/margins": 1.4186785221099854,
      "rewards/rejected": -44.753509521484375,
      "step": 4110
    },
    {
      "epoch": 2.202374979093494,
      "grad_norm": 3.4650077103015228e-09,
      "learning_rate": 2.0011753639026617e-07,
      "logits/chosen": 8.876505851745605,
      "logits/rejected": 8.473733901977539,
      "logps/chosen": -43.526954650878906,
      "logps/rejected": -44.23854446411133,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.526954650878906,
      "rewards/margins": 0.7115898728370667,
      "rewards/rejected": -44.23854446411133,
      "step": 4115
    },
    {
      "epoch": 2.2050510118748954,
      "grad_norm": 1.2031412283640005e-09,
      "learning_rate": 1.988727965115421e-07,
      "logits/chosen": 9.417000770568848,
      "logits/rejected": 8.949925422668457,
      "logps/chosen": -43.354881286621094,
      "logps/rejected": -44.204193115234375,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -43.354881286621094,
      "rewards/margins": 0.8493131399154663,
      "rewards/rejected": -44.204193115234375,
      "step": 4120
    },
    {
      "epoch": 2.207727044656297,
      "grad_norm": 3.2683891641049463e-09,
      "learning_rate": 1.9763097817026713e-07,
      "logits/chosen": 9.431020736694336,
      "logits/rejected": 8.648829460144043,
      "logps/chosen": -43.11323165893555,
      "logps/rejected": -44.73983383178711,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -43.11323165893555,
      "rewards/margins": 1.6266052722930908,
      "rewards/rejected": -44.73983383178711,
      "step": 4125
    },
    {
      "epoch": 2.210403077437699,
      "grad_norm": 7.784828062105631e-12,
      "learning_rate": 1.9639209341456796e-07,
      "logits/chosen": 8.925825119018555,
      "logits/rejected": 8.671640396118164,
      "logps/chosen": -43.275718688964844,
      "logps/rejected": -44.03488540649414,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.275718688964844,
      "rewards/margins": 0.7591708302497864,
      "rewards/rejected": -44.03488540649414,
      "step": 4130
    },
    {
      "epoch": 2.2130791102191,
      "grad_norm": 2.4506430999343263e-05,
      "learning_rate": 1.951561542641102e-07,
      "logits/chosen": 8.659500122070312,
      "logits/rejected": 8.84306812286377,
      "logps/chosen": -43.804786682128906,
      "logps/rejected": -43.858951568603516,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.804786682128906,
      "rewards/margins": 0.0541643388569355,
      "rewards/rejected": -43.858951568603516,
      "step": 4135
    },
    {
      "epoch": 2.215755143000502,
      "grad_norm": 9.560065389393075e-10,
      "learning_rate": 1.939231727099806e-07,
      "logits/chosen": 9.69888973236084,
      "logits/rejected": 9.543991088867188,
      "logps/chosen": -42.596351623535156,
      "logps/rejected": -43.82774353027344,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.596351623535156,
      "rewards/margins": 1.23139226436615,
      "rewards/rejected": -43.82774353027344,
      "step": 4140
    },
    {
      "epoch": 2.2184311757819035,
      "grad_norm": 1.779419934626799e-08,
      "learning_rate": 1.926931607145719e-07,
      "logits/chosen": 8.757194519042969,
      "logits/rejected": 8.255291938781738,
      "logps/chosen": -42.77762222290039,
      "logps/rejected": -44.15127944946289,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.77762222290039,
      "rewards/margins": 1.3736575841903687,
      "rewards/rejected": -44.15127944946289,
      "step": 4145
    },
    {
      "epoch": 2.221107208563305,
      "grad_norm": 1.1968425553405376e-11,
      "learning_rate": 1.9146613021146564e-07,
      "logits/chosen": 9.280985832214355,
      "logits/rejected": 8.816904067993164,
      "logps/chosen": -43.721435546875,
      "logps/rejected": -45.39226531982422,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.721435546875,
      "rewards/margins": 1.670829176902771,
      "rewards/rejected": -45.39226531982422,
      "step": 4150
    },
    {
      "epoch": 2.2237832413447065,
      "grad_norm": 4.360179443915149e-09,
      "learning_rate": 1.9024209310531736e-07,
      "logits/chosen": 9.317049026489258,
      "logits/rejected": 9.267797470092773,
      "logps/chosen": -43.670799255371094,
      "logps/rejected": -43.69342041015625,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -43.670799255371094,
      "rewards/margins": 0.02261648140847683,
      "rewards/rejected": -43.69342041015625,
      "step": 4155
    },
    {
      "epoch": 2.2264592741261082,
      "grad_norm": 1.133369610360667e-05,
      "learning_rate": 1.890210612717401e-07,
      "logits/chosen": 9.052912712097168,
      "logits/rejected": 8.901524543762207,
      "logps/chosen": -42.55875778198242,
      "logps/rejected": -44.060935974121094,
      "loss": 0.0004,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.55875778198242,
      "rewards/margins": 1.502176284790039,
      "rewards/rejected": -44.060935974121094,
      "step": 4160
    },
    {
      "epoch": 2.2291353069075095,
      "grad_norm": 4.6609431491087264e-11,
      "learning_rate": 1.8780304655719054e-07,
      "logits/chosen": 9.408945083618164,
      "logits/rejected": 8.934088706970215,
      "logps/chosen": -43.247955322265625,
      "logps/rejected": -44.729820251464844,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.247955322265625,
      "rewards/margins": 1.481859564781189,
      "rewards/rejected": -44.729820251464844,
      "step": 4165
    },
    {
      "epoch": 2.231811339688911,
      "grad_norm": 8.508534257493168e-12,
      "learning_rate": 1.865880607788523e-07,
      "logits/chosen": 8.797937393188477,
      "logits/rejected": 8.62280559539795,
      "logps/chosen": -43.60547637939453,
      "logps/rejected": -44.05706024169922,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -43.60547637939453,
      "rewards/margins": 0.45158448815345764,
      "rewards/rejected": -44.05706024169922,
      "step": 4170
    },
    {
      "epoch": 2.234487372470313,
      "grad_norm": 1.238662880192874e-09,
      "learning_rate": 1.8537611572452316e-07,
      "logits/chosen": 9.218876838684082,
      "logits/rejected": 8.86221981048584,
      "logps/chosen": -42.84282302856445,
      "logps/rejected": -43.70226287841797,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.84282302856445,
      "rewards/margins": 0.859442412853241,
      "rewards/rejected": -43.70226287841797,
      "step": 4175
    },
    {
      "epoch": 2.237163405251714,
      "grad_norm": 3.0667150759492616e-08,
      "learning_rate": 1.84167223152499e-07,
      "logits/chosen": 9.290166854858398,
      "logits/rejected": 8.51961898803711,
      "logps/chosen": -41.8538932800293,
      "logps/rejected": -43.41326904296875,
      "loss": 0.0012,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.8538932800293,
      "rewards/margins": 1.559377670288086,
      "rewards/rejected": -43.41326904296875,
      "step": 4180
    },
    {
      "epoch": 2.239839438033116,
      "grad_norm": 3.211804441308766e-10,
      "learning_rate": 1.8296139479146112e-07,
      "logits/chosen": 9.022268295288086,
      "logits/rejected": 9.100817680358887,
      "logps/chosen": -43.53455352783203,
      "logps/rejected": -43.85361099243164,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -43.53455352783203,
      "rewards/margins": 0.31906184554100037,
      "rewards/rejected": -43.85361099243164,
      "step": 4185
    },
    {
      "epoch": 2.2425154708145176,
      "grad_norm": 8.513984700052057e-12,
      "learning_rate": 1.8175864234036132e-07,
      "logits/chosen": 8.85918140411377,
      "logits/rejected": 8.641780853271484,
      "logps/chosen": -43.27741241455078,
      "logps/rejected": -44.291725158691406,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.27741241455078,
      "rewards/margins": 1.014306664466858,
      "rewards/rejected": -44.291725158691406,
      "step": 4190
    },
    {
      "epoch": 2.245191503595919,
      "grad_norm": 3.601210554763539e-11,
      "learning_rate": 1.805589774683094e-07,
      "logits/chosen": 9.456896781921387,
      "logits/rejected": 9.032327651977539,
      "logps/chosen": -43.820228576660156,
      "logps/rejected": -44.67981719970703,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.820228576660156,
      "rewards/margins": 0.8595897555351257,
      "rewards/rejected": -44.67981719970703,
      "step": 4195
    },
    {
      "epoch": 2.2478675363773206,
      "grad_norm": 2.2462054780508733e-09,
      "learning_rate": 1.79362411814459e-07,
      "logits/chosen": 8.816685676574707,
      "logits/rejected": 8.74372673034668,
      "logps/chosen": -43.02033233642578,
      "logps/rejected": -43.57416915893555,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -43.02033233642578,
      "rewards/margins": 0.5538488626480103,
      "rewards/rejected": -43.57416915893555,
      "step": 4200
    },
    {
      "epoch": 2.2505435691587223,
      "grad_norm": 1.0210239043090519e-10,
      "learning_rate": 1.7816895698789552e-07,
      "logits/chosen": 9.066399574279785,
      "logits/rejected": 9.013555526733398,
      "logps/chosen": -42.33242416381836,
      "logps/rejected": -44.05732345581055,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.33242416381836,
      "rewards/margins": 1.724898099899292,
      "rewards/rejected": -44.05732345581055,
      "step": 4205
    },
    {
      "epoch": 2.2532196019401236,
      "grad_norm": 3.3869634917082e-11,
      "learning_rate": 1.7697862456752271e-07,
      "logits/chosen": 9.118454933166504,
      "logits/rejected": 8.773681640625,
      "logps/chosen": -42.39700698852539,
      "logps/rejected": -43.76935577392578,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.39700698852539,
      "rewards/margins": 1.372349500656128,
      "rewards/rejected": -43.76935577392578,
      "step": 4210
    },
    {
      "epoch": 2.2558956347215253,
      "grad_norm": 7.532527352209602e-06,
      "learning_rate": 1.7579142610195124e-07,
      "logits/chosen": 8.82850456237793,
      "logits/rejected": 8.392868041992188,
      "logps/chosen": -43.370033264160156,
      "logps/rejected": -44.02642059326172,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.370033264160156,
      "rewards/margins": 0.6563886404037476,
      "rewards/rejected": -44.02642059326172,
      "step": 4215
    },
    {
      "epoch": 2.258571667502927,
      "grad_norm": 4.5320614551379607e-10,
      "learning_rate": 1.7460737310938568e-07,
      "logits/chosen": 9.47398567199707,
      "logits/rejected": 8.70628547668457,
      "logps/chosen": -43.42267990112305,
      "logps/rejected": -44.04685592651367,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.42267990112305,
      "rewards/margins": 0.6241763830184937,
      "rewards/rejected": -44.04685592651367,
      "step": 4220
    },
    {
      "epoch": 2.2612477002843283,
      "grad_norm": 2.3023732778125845e-10,
      "learning_rate": 1.734264770775133e-07,
      "logits/chosen": 9.031586647033691,
      "logits/rejected": 8.20925521850586,
      "logps/chosen": -42.42650604248047,
      "logps/rejected": -44.3620719909668,
      "loss": 0.0002,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.42650604248047,
      "rewards/margins": 1.9355676174163818,
      "rewards/rejected": -44.3620719909668,
      "step": 4225
    },
    {
      "epoch": 2.26392373306573,
      "grad_norm": 1.939435168885417e-09,
      "learning_rate": 1.7224874946339241e-07,
      "logits/chosen": 9.228693008422852,
      "logits/rejected": 8.812799453735352,
      "logps/chosen": -42.75141525268555,
      "logps/rejected": -44.04648971557617,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -42.75141525268555,
      "rewards/margins": 1.295072317123413,
      "rewards/rejected": -44.04648971557617,
      "step": 4230
    },
    {
      "epoch": 2.2665997658471317,
      "grad_norm": 3.402054977933239e-10,
      "learning_rate": 1.7107420169334186e-07,
      "logits/chosen": 9.116961479187012,
      "logits/rejected": 8.883676528930664,
      "logps/chosen": -43.965187072753906,
      "logps/rejected": -44.140235900878906,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.965187072753906,
      "rewards/margins": 0.17505022883415222,
      "rewards/rejected": -44.140235900878906,
      "step": 4235
    },
    {
      "epoch": 2.269275798628533,
      "grad_norm": 2.1772358639588033e-08,
      "learning_rate": 1.6990284516282893e-07,
      "logits/chosen": 9.122404098510742,
      "logits/rejected": 8.788342475891113,
      "logps/chosen": -44.38725662231445,
      "logps/rejected": -45.855464935302734,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -44.38725662231445,
      "rewards/margins": 1.4682106971740723,
      "rewards/rejected": -45.855464935302734,
      "step": 4240
    },
    {
      "epoch": 2.2719518314099347,
      "grad_norm": 1.1436833892050929e-06,
      "learning_rate": 1.687346912363602e-07,
      "logits/chosen": 9.333844184875488,
      "logits/rejected": 8.777189254760742,
      "logps/chosen": -43.29682922363281,
      "logps/rejected": -44.67272186279297,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.29682922363281,
      "rewards/margins": 1.3758885860443115,
      "rewards/rejected": -44.67272186279297,
      "step": 4245
    },
    {
      "epoch": 2.2746278641913364,
      "grad_norm": 2.291591414056221e-06,
      "learning_rate": 1.675697512473697e-07,
      "logits/chosen": 8.850008010864258,
      "logits/rejected": 8.159396171569824,
      "logps/chosen": -42.38923645019531,
      "logps/rejected": -43.2281379699707,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.38923645019531,
      "rewards/margins": 0.8389037847518921,
      "rewards/rejected": -43.2281379699707,
      "step": 4250
    },
    {
      "epoch": 2.2773038969727377,
      "grad_norm": 3.099846744219099e-05,
      "learning_rate": 1.6640803649811087e-07,
      "logits/chosen": 8.921536445617676,
      "logits/rejected": 7.920711517333984,
      "logps/chosen": -41.776832580566406,
      "logps/rejected": -43.69265365600586,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -41.776832580566406,
      "rewards/margins": 1.9158248901367188,
      "rewards/rejected": -43.69265365600586,
      "step": 4255
    },
    {
      "epoch": 2.2799799297541394,
      "grad_norm": 8.506790364959634e-11,
      "learning_rate": 1.6524955825954472e-07,
      "logits/chosen": 9.039824485778809,
      "logits/rejected": 8.914033889770508,
      "logps/chosen": -43.91550827026367,
      "logps/rejected": -44.114463806152344,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -43.91550827026367,
      "rewards/margins": 0.19895082712173462,
      "rewards/rejected": -44.114463806152344,
      "step": 4260
    },
    {
      "epoch": 2.282655962535541,
      "grad_norm": 7.449668752509071e-11,
      "learning_rate": 1.6409432777123277e-07,
      "logits/chosen": 9.453493118286133,
      "logits/rejected": 8.80844783782959,
      "logps/chosen": -42.1513786315918,
      "logps/rejected": -43.998191833496094,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -42.1513786315918,
      "rewards/margins": 1.846813440322876,
      "rewards/rejected": -43.998191833496094,
      "step": 4265
    },
    {
      "epoch": 2.285331995316943,
      "grad_norm": 5.070532250693957e-07,
      "learning_rate": 1.6294235624122577e-07,
      "logits/chosen": 8.8313627243042,
      "logits/rejected": 8.03431510925293,
      "logps/chosen": -41.363250732421875,
      "logps/rejected": -43.45961380004883,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -41.363250732421875,
      "rewards/margins": 2.096357822418213,
      "rewards/rejected": -43.45961380004883,
      "step": 4270
    },
    {
      "epoch": 2.288008028098344,
      "grad_norm": 3.97156346995716e-05,
      "learning_rate": 1.6179365484595697e-07,
      "logits/chosen": 9.09044361114502,
      "logits/rejected": 8.647317886352539,
      "logps/chosen": -42.99861526489258,
      "logps/rejected": -44.119056701660156,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -42.99861526489258,
      "rewards/margins": 1.1204428672790527,
      "rewards/rejected": -44.119056701660156,
      "step": 4275
    },
    {
      "epoch": 2.290684060879746,
      "grad_norm": 6.238097462577515e-07,
      "learning_rate": 1.60648234730132e-07,
      "logits/chosen": 9.191560745239258,
      "logits/rejected": 8.795047760009766,
      "logps/chosen": -42.10997772216797,
      "logps/rejected": -43.478328704833984,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.10997772216797,
      "rewards/margins": 1.3683528900146484,
      "rewards/rejected": -43.478328704833984,
      "step": 4280
    },
    {
      "epoch": 2.293360093661147,
      "grad_norm": 1.7994462064748028e-06,
      "learning_rate": 1.595061070066222e-07,
      "logits/chosen": 9.007406234741211,
      "logits/rejected": 9.154937744140625,
      "logps/chosen": -43.635414123535156,
      "logps/rejected": -43.90370559692383,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -43.635414123535156,
      "rewards/margins": 0.26829349994659424,
      "rewards/rejected": -43.90370559692383,
      "step": 4285
    },
    {
      "epoch": 2.296036126442549,
      "grad_norm": 3.331899347058868e-08,
      "learning_rate": 1.5836728275635542e-07,
      "logits/chosen": 9.42839241027832,
      "logits/rejected": 8.707287788391113,
      "logps/chosen": -43.73933029174805,
      "logps/rejected": -45.089637756347656,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -43.73933029174805,
      "rewards/margins": 1.3503055572509766,
      "rewards/rejected": -45.089637756347656,
      "step": 4290
    },
    {
      "epoch": 2.2987121592239506,
      "grad_norm": 1.1261952875606925e-11,
      "learning_rate": 1.5723177302820984e-07,
      "logits/chosen": 9.148388862609863,
      "logits/rejected": 8.937200546264648,
      "logps/chosen": -42.85658645629883,
      "logps/rejected": -44.64931106567383,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.85658645629883,
      "rewards/margins": 1.7927252054214478,
      "rewards/rejected": -44.64931106567383,
      "step": 4295
    },
    {
      "epoch": 2.3013881920053523,
      "grad_norm": 5.144613684375694e-10,
      "learning_rate": 1.5609958883890544e-07,
      "logits/chosen": 8.734636306762695,
      "logits/rejected": 8.355271339416504,
      "logps/chosen": -42.69615173339844,
      "logps/rejected": -43.04405975341797,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.69615173339844,
      "rewards/margins": 0.34790679812431335,
      "rewards/rejected": -43.04405975341797,
      "step": 4300
    },
    {
      "epoch": 2.3040642247867535,
      "grad_norm": 1.9076431694630282e-13,
      "learning_rate": 1.5497074117289865e-07,
      "logits/chosen": 9.247693061828613,
      "logits/rejected": 8.819948196411133,
      "logps/chosen": -43.84877014160156,
      "logps/rejected": -44.492122650146484,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.84877014160156,
      "rewards/margins": 0.6433536410331726,
      "rewards/rejected": -44.492122650146484,
      "step": 4305
    },
    {
      "epoch": 2.3067402575681553,
      "grad_norm": 7.636399860451499e-08,
      "learning_rate": 1.5384524098227402e-07,
      "logits/chosen": 9.422225952148438,
      "logits/rejected": 8.724987983703613,
      "logps/chosen": -43.411964416503906,
      "logps/rejected": -44.942100524902344,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -43.411964416503906,
      "rewards/margins": 1.530137300491333,
      "rewards/rejected": -44.942100524902344,
      "step": 4310
    },
    {
      "epoch": 2.3094162903495565,
      "grad_norm": 3.4998625370364907e-09,
      "learning_rate": 1.5272309918663974e-07,
      "logits/chosen": 9.082725524902344,
      "logits/rejected": 8.595535278320312,
      "logps/chosen": -43.05567932128906,
      "logps/rejected": -44.44702911376953,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.05567932128906,
      "rewards/margins": 1.3913497924804688,
      "rewards/rejected": -44.44702911376953,
      "step": 4315
    },
    {
      "epoch": 2.3120923231309582,
      "grad_norm": 2.6689369136725444e-09,
      "learning_rate": 1.516043266730201e-07,
      "logits/chosen": 9.244811058044434,
      "logits/rejected": 8.717084884643555,
      "logps/chosen": -43.39812088012695,
      "logps/rejected": -44.62095260620117,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.39812088012695,
      "rewards/margins": 1.2228361368179321,
      "rewards/rejected": -44.62095260620117,
      "step": 4320
    },
    {
      "epoch": 2.31476835591236,
      "grad_norm": 4.100455603489451e-08,
      "learning_rate": 1.504889342957512e-07,
      "logits/chosen": 9.180676460266113,
      "logits/rejected": 8.67148208618164,
      "logps/chosen": -43.18669891357422,
      "logps/rejected": -44.49002456665039,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.18669891357422,
      "rewards/margins": 1.3033275604248047,
      "rewards/rejected": -44.49002456665039,
      "step": 4325
    },
    {
      "epoch": 2.3174443886937617,
      "grad_norm": 1.5867202340473305e-07,
      "learning_rate": 1.4937693287637453e-07,
      "logits/chosen": 9.144622802734375,
      "logits/rejected": 8.766639709472656,
      "logps/chosen": -43.558021545410156,
      "logps/rejected": -44.061065673828125,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.558021545410156,
      "rewards/margins": 0.5030478835105896,
      "rewards/rejected": -44.061065673828125,
      "step": 4330
    },
    {
      "epoch": 2.320120421475163,
      "grad_norm": 1.4873206498981649e-10,
      "learning_rate": 1.4826833320353305e-07,
      "logits/chosen": 9.09286880493164,
      "logits/rejected": 8.727540969848633,
      "logps/chosen": -42.87596130371094,
      "logps/rejected": -43.59950256347656,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.87596130371094,
      "rewards/margins": 0.7235434651374817,
      "rewards/rejected": -43.59950256347656,
      "step": 4335
    },
    {
      "epoch": 2.3227964542565647,
      "grad_norm": 2.27176365555468e-08,
      "learning_rate": 1.4716314603286528e-07,
      "logits/chosen": 9.23253345489502,
      "logits/rejected": 8.481476783752441,
      "logps/chosen": -42.807376861572266,
      "logps/rejected": -44.712669372558594,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.807376861572266,
      "rewards/margins": 1.9052963256835938,
      "rewards/rejected": -44.712669372558594,
      "step": 4340
    },
    {
      "epoch": 2.3254724870379664,
      "grad_norm": 1.2941480635934895e-07,
      "learning_rate": 1.4606138208690233e-07,
      "logits/chosen": 8.741349220275879,
      "logits/rejected": 8.724069595336914,
      "logps/chosen": -43.336788177490234,
      "logps/rejected": -44.0677490234375,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.336788177490234,
      "rewards/margins": 0.7309597730636597,
      "rewards/rejected": -44.0677490234375,
      "step": 4345
    },
    {
      "epoch": 2.3281485198193677,
      "grad_norm": 2.2287024403415583e-11,
      "learning_rate": 1.4496305205496251e-07,
      "logits/chosen": 8.90785026550293,
      "logits/rejected": 8.737528800964355,
      "logps/chosen": -44.29743194580078,
      "logps/rejected": -44.66332244873047,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.29743194580078,
      "rewards/margins": 0.36587873101234436,
      "rewards/rejected": -44.66332244873047,
      "step": 4350
    },
    {
      "epoch": 2.3308245526007694,
      "grad_norm": 8.668890570301739e-09,
      "learning_rate": 1.4386816659304895e-07,
      "logits/chosen": 9.492953300476074,
      "logits/rejected": 8.876031875610352,
      "logps/chosen": -44.036006927490234,
      "logps/rejected": -44.11336898803711,
      "loss": 0.0011,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -44.036006927490234,
      "rewards/margins": 0.07736074924468994,
      "rewards/rejected": -44.11336898803711,
      "step": 4355
    },
    {
      "epoch": 2.333500585382171,
      "grad_norm": 2.262696609355777e-05,
      "learning_rate": 1.4277673632374492e-07,
      "logits/chosen": 9.042167663574219,
      "logits/rejected": 8.144396781921387,
      "logps/chosen": -42.3798828125,
      "logps/rejected": -44.473358154296875,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -42.3798828125,
      "rewards/margins": 2.093472957611084,
      "rewards/rejected": -44.473358154296875,
      "step": 4360
    },
    {
      "epoch": 2.3361766181635724,
      "grad_norm": 8.912362764443241e-06,
      "learning_rate": 1.416887718361119e-07,
      "logits/chosen": 9.104842185974121,
      "logits/rejected": 9.123126029968262,
      "logps/chosen": -44.14275360107422,
      "logps/rejected": -45.2035026550293,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -44.14275360107422,
      "rewards/margins": 1.0607534646987915,
      "rewards/rejected": -45.2035026550293,
      "step": 4365
    },
    {
      "epoch": 2.338852650944974,
      "grad_norm": 4.064646509395895e-11,
      "learning_rate": 1.406042836855859e-07,
      "logits/chosen": 9.455392837524414,
      "logits/rejected": 9.015202522277832,
      "logps/chosen": -44.02656555175781,
      "logps/rejected": -44.52220916748047,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -44.02656555175781,
      "rewards/margins": 0.4956429898738861,
      "rewards/rejected": -44.52220916748047,
      "step": 4370
    },
    {
      "epoch": 2.341528683726376,
      "grad_norm": 1.937515268137528e-07,
      "learning_rate": 1.3952328239387595e-07,
      "logits/chosen": 9.14968490600586,
      "logits/rejected": 8.300264358520508,
      "logps/chosen": -42.67992401123047,
      "logps/rejected": -43.92139434814453,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.67992401123047,
      "rewards/margins": 1.2414720058441162,
      "rewards/rejected": -43.92139434814453,
      "step": 4375
    },
    {
      "epoch": 2.344204716507777,
      "grad_norm": 1.4018277863105718e-08,
      "learning_rate": 1.3844577844886109e-07,
      "logits/chosen": 9.254443168640137,
      "logits/rejected": 8.511683464050293,
      "logps/chosen": -41.791778564453125,
      "logps/rejected": -43.45569610595703,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.791778564453125,
      "rewards/margins": 1.6639143228530884,
      "rewards/rejected": -43.45569610595703,
      "step": 4380
    },
    {
      "epoch": 2.346880749289179,
      "grad_norm": 1.051836029427001e-12,
      "learning_rate": 1.3737178230448955e-07,
      "logits/chosen": 8.885170936584473,
      "logits/rejected": 8.619671821594238,
      "logps/chosen": -43.519752502441406,
      "logps/rejected": -44.45296859741211,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.519752502441406,
      "rewards/margins": 0.9332124590873718,
      "rewards/rejected": -44.45296859741211,
      "step": 4385
    },
    {
      "epoch": 2.3495567820705805,
      "grad_norm": 6.280604241054e-13,
      "learning_rate": 1.363013043806764e-07,
      "logits/chosen": 9.265891075134277,
      "logits/rejected": 8.933459281921387,
      "logps/chosen": -43.056278228759766,
      "logps/rejected": -44.31239318847656,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.056278228759766,
      "rewards/margins": 1.256115198135376,
      "rewards/rejected": -44.31239318847656,
      "step": 4390
    },
    {
      "epoch": 2.3522328148519818,
      "grad_norm": 1.1861349984204927e-08,
      "learning_rate": 1.352343550632034e-07,
      "logits/chosen": 9.280359268188477,
      "logits/rejected": 8.73161506652832,
      "logps/chosen": -42.6974983215332,
      "logps/rejected": -44.89132308959961,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.6974983215332,
      "rewards/margins": 2.1938223838806152,
      "rewards/rejected": -44.89132308959961,
      "step": 4395
    },
    {
      "epoch": 2.3549088476333835,
      "grad_norm": 1.2432950582382938e-09,
      "learning_rate": 1.3417094470361722e-07,
      "logits/chosen": 9.069659233093262,
      "logits/rejected": 8.624251365661621,
      "logps/chosen": -42.985801696777344,
      "logps/rejected": -44.3646125793457,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.985801696777344,
      "rewards/margins": 1.378806233406067,
      "rewards/rejected": -44.3646125793457,
      "step": 4400
    },
    {
      "epoch": 2.3549088476333835,
      "eval_logits/chosen": 7.58363676071167,
      "eval_logits/rejected": 7.291767597198486,
      "eval_logps/chosen": -42.94476318359375,
      "eval_logps/rejected": -44.052490234375,
      "eval_loss": 2.09151721719536e-07,
      "eval_rewards/accuracies": 0.5704748034477234,
      "eval_rewards/chosen": -42.94476318359375,
      "eval_rewards/margins": 1.1077219247817993,
      "eval_rewards/rejected": -44.052490234375,
      "eval_runtime": 40.5432,
      "eval_samples_per_second": 33.174,
      "eval_steps_per_second": 8.312,
      "step": 4400
    },
    {
      "epoch": 2.357584880414785,
      "grad_norm": 1.137727206451965e-07,
      "learning_rate": 1.3311108361913015e-07,
      "logits/chosen": 9.142969131469727,
      "logits/rejected": 9.181938171386719,
      "logps/chosen": -44.06669235229492,
      "logps/rejected": -44.470558166503906,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.06669235229492,
      "rewards/margins": 0.40386858582496643,
      "rewards/rejected": -44.470558166503906,
      "step": 4405
    },
    {
      "epoch": 2.3602609131961865,
      "grad_norm": 1.2430418719987272e-06,
      "learning_rate": 1.3205478209251874e-07,
      "logits/chosen": 9.302844047546387,
      "logits/rejected": 9.121850967407227,
      "logps/chosen": -43.989173889160156,
      "logps/rejected": -44.57482147216797,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.989173889160156,
      "rewards/margins": 0.5856438875198364,
      "rewards/rejected": -44.57482147216797,
      "step": 4410
    },
    {
      "epoch": 2.362936945977588,
      "grad_norm": 1.6140402659209794e-08,
      "learning_rate": 1.310020503720254e-07,
      "logits/chosen": 8.872774124145508,
      "logits/rejected": 8.172953605651855,
      "logps/chosen": -42.81657028198242,
      "logps/rejected": -44.09730911254883,
      "loss": 0.0002,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.81657028198242,
      "rewards/margins": 1.2807376384735107,
      "rewards/rejected": -44.09730911254883,
      "step": 4415
    },
    {
      "epoch": 2.36561297875899,
      "grad_norm": 3.5868902326904634e-12,
      "learning_rate": 1.2995289867125752e-07,
      "logits/chosen": 9.153031349182129,
      "logits/rejected": 8.872515678405762,
      "logps/chosen": -44.24485778808594,
      "logps/rejected": -45.25538635253906,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.24485778808594,
      "rewards/margins": 1.0105345249176025,
      "rewards/rejected": -45.25538635253906,
      "step": 4420
    },
    {
      "epoch": 2.368289011540391,
      "grad_norm": 4.363046327658823e-09,
      "learning_rate": 1.2890733716908986e-07,
      "logits/chosen": 8.967996597290039,
      "logits/rejected": 9.00118637084961,
      "logps/chosen": -42.25114822387695,
      "logps/rejected": -43.24909591674805,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.25114822387695,
      "rewards/margins": 0.9979490041732788,
      "rewards/rejected": -43.24909591674805,
      "step": 4425
    },
    {
      "epoch": 2.370965044321793,
      "grad_norm": 5.64828628500646e-11,
      "learning_rate": 1.2786537600956454e-07,
      "logits/chosen": 9.335803985595703,
      "logits/rejected": 8.578765869140625,
      "logps/chosen": -42.637569427490234,
      "logps/rejected": -44.36760711669922,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -42.637569427490234,
      "rewards/margins": 1.7300386428833008,
      "rewards/rejected": -44.36760711669922,
      "step": 4430
    },
    {
      "epoch": 2.3736410771031946,
      "grad_norm": 2.7188667113474353e-08,
      "learning_rate": 1.268270253017933e-07,
      "logits/chosen": 9.207649230957031,
      "logits/rejected": 8.488981246948242,
      "logps/chosen": -43.47707748413086,
      "logps/rejected": -44.49448013305664,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.47707748413086,
      "rewards/margins": 1.0174052715301514,
      "rewards/rejected": -44.49448013305664,
      "step": 4435
    },
    {
      "epoch": 2.376317109884596,
      "grad_norm": 2.0235296188135633e-08,
      "learning_rate": 1.257922951198591e-07,
      "logits/chosen": 9.092676162719727,
      "logits/rejected": 8.0695219039917,
      "logps/chosen": -42.53782653808594,
      "logps/rejected": -43.3267822265625,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.53782653808594,
      "rewards/margins": 0.7889548540115356,
      "rewards/rejected": -43.3267822265625,
      "step": 4440
    },
    {
      "epoch": 2.3789931426659976,
      "grad_norm": 5.959831703014962e-09,
      "learning_rate": 1.24761195502719e-07,
      "logits/chosen": 9.072807312011719,
      "logits/rejected": 8.297175407409668,
      "logps/chosen": -43.0941047668457,
      "logps/rejected": -44.660926818847656,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -43.0941047668457,
      "rewards/margins": 1.5668278932571411,
      "rewards/rejected": -44.660926818847656,
      "step": 4445
    },
    {
      "epoch": 2.3816691754473993,
      "grad_norm": 0.0006455802202963956,
      "learning_rate": 1.2373373645410573e-07,
      "logits/chosen": 9.012727737426758,
      "logits/rejected": 8.577821731567383,
      "logps/chosen": -44.019378662109375,
      "logps/rejected": -44.51078414916992,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.019378662109375,
      "rewards/margins": 0.49140363931655884,
      "rewards/rejected": -44.51078414916992,
      "step": 4450
    },
    {
      "epoch": 2.384345208228801,
      "grad_norm": 2.178148998749168e-10,
      "learning_rate": 1.2270992794243175e-07,
      "logits/chosen": 9.32244873046875,
      "logits/rejected": 8.821152687072754,
      "logps/chosen": -44.0055046081543,
      "logps/rejected": -44.87663269042969,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.0055046081543,
      "rewards/margins": 0.8711255192756653,
      "rewards/rejected": -44.87663269042969,
      "step": 4455
    },
    {
      "epoch": 2.3870212410102023,
      "grad_norm": 1.3801590406911339e-08,
      "learning_rate": 1.2168977990069147e-07,
      "logits/chosen": 9.49370288848877,
      "logits/rejected": 8.74947452545166,
      "logps/chosen": -43.95867156982422,
      "logps/rejected": -44.999305725097656,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -43.95867156982422,
      "rewards/margins": 1.0406330823898315,
      "rewards/rejected": -44.999305725097656,
      "step": 4460
    },
    {
      "epoch": 2.389697273791604,
      "grad_norm": 8.824963748286137e-10,
      "learning_rate": 1.206733022263659e-07,
      "logits/chosen": 9.082025527954102,
      "logits/rejected": 8.632207870483398,
      "logps/chosen": -42.01361846923828,
      "logps/rejected": -43.52204895019531,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.01361846923828,
      "rewards/margins": 1.5084317922592163,
      "rewards/rejected": -43.52204895019531,
      "step": 4465
    },
    {
      "epoch": 2.3923733065730053,
      "grad_norm": 1.3673137598535915e-08,
      "learning_rate": 1.1966050478132572e-07,
      "logits/chosen": 9.137206077575684,
      "logits/rejected": 8.792137145996094,
      "logps/chosen": -44.097599029541016,
      "logps/rejected": -44.59095764160156,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -44.097599029541016,
      "rewards/margins": 0.4933616518974304,
      "rewards/rejected": -44.59095764160156,
      "step": 4470
    },
    {
      "epoch": 2.395049339354407,
      "grad_norm": 7.719831477307347e-08,
      "learning_rate": 1.1865139739173635e-07,
      "logits/chosen": 9.21369457244873,
      "logits/rejected": 8.449313163757324,
      "logps/chosen": -45.10253143310547,
      "logps/rejected": -45.31608200073242,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.10253143310547,
      "rewards/margins": 0.21355172991752625,
      "rewards/rejected": -45.31608200073242,
      "step": 4475
    },
    {
      "epoch": 2.3977253721358087,
      "grad_norm": 8.309703240889619e-08,
      "learning_rate": 1.1764598984796187e-07,
      "logits/chosen": 9.55870246887207,
      "logits/rejected": 8.969377517700195,
      "logps/chosen": -43.30122756958008,
      "logps/rejected": -44.269256591796875,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.30122756958008,
      "rewards/margins": 0.9680284261703491,
      "rewards/rejected": -44.269256591796875,
      "step": 4480
    },
    {
      "epoch": 2.4004014049172104,
      "grad_norm": 2.707401359543554e-10,
      "learning_rate": 1.1664429190447095e-07,
      "logits/chosen": 8.745327949523926,
      "logits/rejected": 8.617340087890625,
      "logps/chosen": -42.18756866455078,
      "logps/rejected": -44.07577896118164,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.18756866455078,
      "rewards/margins": 1.888204574584961,
      "rewards/rejected": -44.07577896118164,
      "step": 4485
    },
    {
      "epoch": 2.4030774376986117,
      "grad_norm": 4.184333434134072e-09,
      "learning_rate": 1.1564631327974122e-07,
      "logits/chosen": 9.120786666870117,
      "logits/rejected": 8.588520050048828,
      "logps/chosen": -42.65946578979492,
      "logps/rejected": -44.833744049072266,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -42.65946578979492,
      "rewards/margins": 2.1742758750915527,
      "rewards/rejected": -44.833744049072266,
      "step": 4490
    },
    {
      "epoch": 2.4057534704800134,
      "grad_norm": 9.60038359522924e-10,
      "learning_rate": 1.1465206365616587e-07,
      "logits/chosen": 9.228276252746582,
      "logits/rejected": 8.587491035461426,
      "logps/chosen": -43.101383209228516,
      "logps/rejected": -44.66340255737305,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -43.101383209228516,
      "rewards/margins": 1.562018871307373,
      "rewards/rejected": -44.66340255737305,
      "step": 4495
    },
    {
      "epoch": 2.408429503261415,
      "grad_norm": 2.097560065575246e-12,
      "learning_rate": 1.1366155267995887e-07,
      "logits/chosen": 9.23318099975586,
      "logits/rejected": 9.171361923217773,
      "logps/chosen": -44.490020751953125,
      "logps/rejected": -45.20877456665039,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.490020751953125,
      "rewards/margins": 0.7187555432319641,
      "rewards/rejected": -45.20877456665039,
      "step": 4500
    },
    {
      "epoch": 2.4111055360428164,
      "grad_norm": 6.109008494100467e-06,
      "learning_rate": 1.1267478996106228e-07,
      "logits/chosen": 9.225412368774414,
      "logits/rejected": 8.4717435836792,
      "logps/chosen": -42.46234130859375,
      "logps/rejected": -43.9901123046875,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.46234130859375,
      "rewards/margins": 1.5277734994888306,
      "rewards/rejected": -43.9901123046875,
      "step": 4505
    },
    {
      "epoch": 2.413781568824218,
      "grad_norm": 2.621012448107174e-08,
      "learning_rate": 1.116917850730521e-07,
      "logits/chosen": 9.25502872467041,
      "logits/rejected": 8.600152969360352,
      "logps/chosen": -43.92424392700195,
      "logps/rejected": -43.80561828613281,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -43.92424392700195,
      "rewards/margins": -0.118623748421669,
      "rewards/rejected": -43.80561828613281,
      "step": 4510
    },
    {
      "epoch": 2.41645760160562,
      "grad_norm": 9.10958713801315e-08,
      "learning_rate": 1.1071254755304637e-07,
      "logits/chosen": 8.845125198364258,
      "logits/rejected": 8.52183723449707,
      "logps/chosen": -42.841552734375,
      "logps/rejected": -43.588497161865234,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.841552734375,
      "rewards/margins": 0.7469453811645508,
      "rewards/rejected": -43.588497161865234,
      "step": 4515
    },
    {
      "epoch": 2.419133634387021,
      "grad_norm": 3.761370057397501e-10,
      "learning_rate": 1.0973708690161143e-07,
      "logits/chosen": 9.203384399414062,
      "logits/rejected": 8.749646186828613,
      "logps/chosen": -41.85224151611328,
      "logps/rejected": -43.942543029785156,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -41.85224151611328,
      "rewards/margins": 2.090305805206299,
      "rewards/rejected": -43.942543029785156,
      "step": 4520
    },
    {
      "epoch": 2.421809667168423,
      "grad_norm": 2.1468272897694934e-10,
      "learning_rate": 1.0876541258267119e-07,
      "logits/chosen": 9.081708908081055,
      "logits/rejected": 8.44297981262207,
      "logps/chosen": -42.12873840332031,
      "logps/rejected": -42.96746826171875,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.12873840332031,
      "rewards/margins": 0.8387306332588196,
      "rewards/rejected": -42.96746826171875,
      "step": 4525
    },
    {
      "epoch": 2.4244856999498245,
      "grad_norm": 2.2739414547381716e-06,
      "learning_rate": 1.0779753402341379e-07,
      "logits/chosen": 9.346722602844238,
      "logits/rejected": 8.936014175415039,
      "logps/chosen": -42.54629898071289,
      "logps/rejected": -43.609249114990234,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -42.54629898071289,
      "rewards/margins": 1.0629475116729736,
      "rewards/rejected": -43.609249114990234,
      "step": 4530
    },
    {
      "epoch": 2.427161732731226,
      "grad_norm": 1.3633868479780688e-08,
      "learning_rate": 1.0683346061420157e-07,
      "logits/chosen": 9.418802261352539,
      "logits/rejected": 8.973909378051758,
      "logps/chosen": -43.503509521484375,
      "logps/rejected": -45.15605926513672,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.503509521484375,
      "rewards/margins": 1.6525547504425049,
      "rewards/rejected": -45.15605926513672,
      "step": 4535
    },
    {
      "epoch": 2.4298377655126275,
      "grad_norm": 1.468131527788943e-07,
      "learning_rate": 1.0587320170847874e-07,
      "logits/chosen": 9.299947738647461,
      "logits/rejected": 8.888525009155273,
      "logps/chosen": -44.736328125,
      "logps/rejected": -44.92253112792969,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.736328125,
      "rewards/margins": 0.1862012892961502,
      "rewards/rejected": -44.92253112792969,
      "step": 4540
    },
    {
      "epoch": 2.4325137982940293,
      "grad_norm": 8.094425573062365e-08,
      "learning_rate": 1.0491676662268156e-07,
      "logits/chosen": 9.102094650268555,
      "logits/rejected": 8.735517501831055,
      "logps/chosen": -44.068443298339844,
      "logps/rejected": -45.31291961669922,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.068443298339844,
      "rewards/margins": 1.2444796562194824,
      "rewards/rejected": -45.31291961669922,
      "step": 4545
    },
    {
      "epoch": 2.4351898310754305,
      "grad_norm": 9.565163343963566e-07,
      "learning_rate": 1.0396416463614732e-07,
      "logits/chosen": 8.991244316101074,
      "logits/rejected": 8.889235496520996,
      "logps/chosen": -42.168846130371094,
      "logps/rejected": -43.555091857910156,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.168846130371094,
      "rewards/margins": 1.3862473964691162,
      "rewards/rejected": -43.555091857910156,
      "step": 4550
    },
    {
      "epoch": 2.4378658638568322,
      "grad_norm": 6.569198730149456e-11,
      "learning_rate": 1.0301540499102479e-07,
      "logits/chosen": 8.95669174194336,
      "logits/rejected": 8.926271438598633,
      "logps/chosen": -43.23931121826172,
      "logps/rejected": -44.545623779296875,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.23931121826172,
      "rewards/margins": 1.306312084197998,
      "rewards/rejected": -44.545623779296875,
      "step": 4555
    },
    {
      "epoch": 2.440541896638234,
      "grad_norm": 5.905434562500467e-08,
      "learning_rate": 1.0207049689218405e-07,
      "logits/chosen": 9.12828254699707,
      "logits/rejected": 8.25415325164795,
      "logps/chosen": -42.28589630126953,
      "logps/rejected": -44.01564025878906,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.28589630126953,
      "rewards/margins": 1.7297471761703491,
      "rewards/rejected": -44.01564025878906,
      "step": 4560
    },
    {
      "epoch": 2.4432179294196352,
      "grad_norm": 0.19605552077566635,
      "learning_rate": 1.0112944950712782e-07,
      "logits/chosen": 8.721559524536133,
      "logits/rejected": 8.356966018676758,
      "logps/chosen": -42.18324279785156,
      "logps/rejected": -43.19923400878906,
      "loss": 0.0006,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.18324279785156,
      "rewards/margins": 1.0159876346588135,
      "rewards/rejected": -43.19923400878906,
      "step": 4565
    },
    {
      "epoch": 2.445893962201037,
      "grad_norm": 4.159598733161546e-07,
      "learning_rate": 1.0019227196590174e-07,
      "logits/chosen": 9.34399127960205,
      "logits/rejected": 8.758341789245605,
      "logps/chosen": -43.436161041259766,
      "logps/rejected": -45.19282913208008,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.436161041259766,
      "rewards/margins": 1.7566697597503662,
      "rewards/rejected": -45.19282913208008,
      "step": 4570
    },
    {
      "epoch": 2.4485699949824387,
      "grad_norm": 1.8619680091328555e-11,
      "learning_rate": 9.925897336100664e-08,
      "logits/chosen": 8.596741676330566,
      "logits/rejected": 8.583913803100586,
      "logps/chosen": -44.03644943237305,
      "logps/rejected": -44.815696716308594,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.03644943237305,
      "rewards/margins": 0.7792443037033081,
      "rewards/rejected": -44.815696716308594,
      "step": 4575
    },
    {
      "epoch": 2.45124602776384,
      "grad_norm": 2.2907628883556757e-09,
      "learning_rate": 9.832956274730946e-08,
      "logits/chosen": 8.77738094329834,
      "logits/rejected": 8.592802047729492,
      "logps/chosen": -42.731876373291016,
      "logps/rejected": -44.014312744140625,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.731876373291016,
      "rewards/margins": 1.2824397087097168,
      "rewards/rejected": -44.014312744140625,
      "step": 4580
    },
    {
      "epoch": 2.4539220605452416,
      "grad_norm": 2.1553071307698722e-06,
      "learning_rate": 9.740404914195633e-08,
      "logits/chosen": 9.135787010192871,
      "logits/rejected": 8.799894332885742,
      "logps/chosen": -43.23937225341797,
      "logps/rejected": -43.849246978759766,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.23937225341797,
      "rewards/margins": 0.6098742485046387,
      "rewards/rejected": -43.849246978759766,
      "step": 4585
    },
    {
      "epoch": 2.4565980933266434,
      "grad_norm": 2.478421466199099e-09,
      "learning_rate": 9.648244152428392e-08,
      "logits/chosen": 9.190836906433105,
      "logits/rejected": 8.90406608581543,
      "logps/chosen": -43.241981506347656,
      "logps/rejected": -44.98204040527344,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.241981506347656,
      "rewards/margins": 1.740060567855835,
      "rewards/rejected": -44.98204040527344,
      "step": 4590
    },
    {
      "epoch": 2.4592741261080446,
      "grad_norm": 1.902457079906106e-06,
      "learning_rate": 9.556474883573379e-08,
      "logits/chosen": 9.186389923095703,
      "logits/rejected": 8.845205307006836,
      "logps/chosen": -43.44452667236328,
      "logps/rejected": -45.01343536376953,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.44452667236328,
      "rewards/margins": 1.5689066648483276,
      "rewards/rejected": -45.01343536376953,
      "step": 4595
    },
    {
      "epoch": 2.4619501588894463,
      "grad_norm": 4.090602261307582e-13,
      "learning_rate": 9.465097997976412e-08,
      "logits/chosen": 9.1355562210083,
      "logits/rejected": 8.413407325744629,
      "logps/chosen": -42.12215042114258,
      "logps/rejected": -43.92107009887695,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.12215042114258,
      "rewards/margins": 1.79892098903656,
      "rewards/rejected": -43.92107009887695,
      "step": 4600
    },
    {
      "epoch": 2.464626191670848,
      "grad_norm": 5.643186007673075e-09,
      "learning_rate": 9.374114382176457e-08,
      "logits/chosen": 9.422418594360352,
      "logits/rejected": 8.689146041870117,
      "logps/chosen": -43.20514678955078,
      "logps/rejected": -43.85572052001953,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -43.20514678955078,
      "rewards/margins": 0.6505736112594604,
      "rewards/rejected": -43.85572052001953,
      "step": 4605
    },
    {
      "epoch": 2.46730222445225,
      "grad_norm": 0.0005032262561090332,
      "learning_rate": 9.283524918896945e-08,
      "logits/chosen": 9.425394058227539,
      "logits/rejected": 8.85915470123291,
      "logps/chosen": -43.71038818359375,
      "logps/rejected": -45.225032806396484,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.71038818359375,
      "rewards/margins": 1.5146474838256836,
      "rewards/rejected": -45.225032806396484,
      "step": 4610
    },
    {
      "epoch": 2.469978257233651,
      "grad_norm": 7.370446268456402e-13,
      "learning_rate": 9.193330487037232e-08,
      "logits/chosen": 9.175050735473633,
      "logits/rejected": 8.738815307617188,
      "logps/chosen": -43.13275909423828,
      "logps/rejected": -44.79536819458008,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -43.13275909423828,
      "rewards/margins": 1.6626050472259521,
      "rewards/rejected": -44.79536819458008,
      "step": 4615
    },
    {
      "epoch": 2.4726542900150528,
      "grad_norm": 2.1216955098910236e-13,
      "learning_rate": 9.103531961664118e-08,
      "logits/chosen": 8.888827323913574,
      "logits/rejected": 8.366586685180664,
      "logps/chosen": -43.796363830566406,
      "logps/rejected": -45.556129455566406,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.796363830566406,
      "rewards/margins": 1.7597652673721313,
      "rewards/rejected": -45.556129455566406,
      "step": 4620
    },
    {
      "epoch": 2.475330322796454,
      "grad_norm": 1.4830532008993502e-10,
      "learning_rate": 9.014130214003269e-08,
      "logits/chosen": 8.817378044128418,
      "logits/rejected": 8.828986167907715,
      "logps/chosen": -42.25873565673828,
      "logps/rejected": -43.57822799682617,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.25873565673828,
      "rewards/margins": 1.319486141204834,
      "rewards/rejected": -43.57822799682617,
      "step": 4625
    },
    {
      "epoch": 2.4780063555778558,
      "grad_norm": 3.854193146921064e-08,
      "learning_rate": 8.925126111430848e-08,
      "logits/chosen": 8.268295288085938,
      "logits/rejected": 8.089634895324707,
      "logps/chosen": -42.152252197265625,
      "logps/rejected": -43.164939880371094,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -42.152252197265625,
      "rewards/margins": 1.0126903057098389,
      "rewards/rejected": -43.164939880371094,
      "step": 4630
    },
    {
      "epoch": 2.4806823883592575,
      "grad_norm": 6.156027836353993e-08,
      "learning_rate": 8.83652051746504e-08,
      "logits/chosen": 9.214881896972656,
      "logits/rejected": 8.371711730957031,
      "logps/chosen": -43.5845947265625,
      "logps/rejected": -43.76476287841797,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.5845947265625,
      "rewards/margins": 0.18017558753490448,
      "rewards/rejected": -43.76476287841797,
      "step": 4635
    },
    {
      "epoch": 2.483358421140659,
      "grad_norm": 2.1780720295974752e-11,
      "learning_rate": 8.748314291757696e-08,
      "logits/chosen": 8.744614601135254,
      "logits/rejected": 8.606954574584961,
      "logps/chosen": -42.13679122924805,
      "logps/rejected": -44.540260314941406,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.13679122924805,
      "rewards/margins": 2.4034669399261475,
      "rewards/rejected": -44.540260314941406,
      "step": 4640
    },
    {
      "epoch": 2.4860344539220605,
      "grad_norm": 0.00046056731037930645,
      "learning_rate": 8.660508290086032e-08,
      "logits/chosen": 9.15407657623291,
      "logits/rejected": 8.441764831542969,
      "logps/chosen": -42.433143615722656,
      "logps/rejected": -43.31450653076172,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.433143615722656,
      "rewards/margins": 0.8813636898994446,
      "rewards/rejected": -43.31450653076172,
      "step": 4645
    },
    {
      "epoch": 2.488710486703462,
      "grad_norm": 1.4789977070032176e-09,
      "learning_rate": 8.573103364344231e-08,
      "logits/chosen": 9.336080551147461,
      "logits/rejected": 8.67591381072998,
      "logps/chosen": -42.2813835144043,
      "logps/rejected": -44.98908615112305,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -42.2813835144043,
      "rewards/margins": 2.7077033519744873,
      "rewards/rejected": -44.98908615112305,
      "step": 4650
    },
    {
      "epoch": 2.4913865194848634,
      "grad_norm": 1.0095563167156927e-10,
      "learning_rate": 8.486100362535292e-08,
      "logits/chosen": 9.016302108764648,
      "logits/rejected": 8.631976127624512,
      "logps/chosen": -42.29338836669922,
      "logps/rejected": -43.491416931152344,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.29338836669922,
      "rewards/margins": 1.198028326034546,
      "rewards/rejected": -43.491416931152344,
      "step": 4655
    },
    {
      "epoch": 2.494062552266265,
      "grad_norm": 1.1782215295645974e-05,
      "learning_rate": 8.399500128762693e-08,
      "logits/chosen": 8.841638565063477,
      "logits/rejected": 8.438671112060547,
      "logps/chosen": -43.53633117675781,
      "logps/rejected": -44.23197555541992,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -43.53633117675781,
      "rewards/margins": 0.6956467628479004,
      "rewards/rejected": -44.23197555541992,
      "step": 4660
    },
    {
      "epoch": 2.496738585047667,
      "grad_norm": 1.3249847462314841e-08,
      "learning_rate": 8.313303503222313e-08,
      "logits/chosen": 9.326006889343262,
      "logits/rejected": 8.895910263061523,
      "logps/chosen": -43.320247650146484,
      "logps/rejected": -42.704132080078125,
      "loss": 0.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -43.320247650146484,
      "rewards/margins": -0.6161131858825684,
      "rewards/rejected": -42.704132080078125,
      "step": 4665
    },
    {
      "epoch": 2.4994146178290686,
      "grad_norm": 3.7773475599607546e-07,
      "learning_rate": 8.227511322194164e-08,
      "logits/chosen": 9.266204833984375,
      "logits/rejected": 8.7720947265625,
      "logps/chosen": -42.21365737915039,
      "logps/rejected": -43.435455322265625,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.21365737915039,
      "rewards/margins": 1.2218005657196045,
      "rewards/rejected": -43.435455322265625,
      "step": 4670
    },
    {
      "epoch": 2.50209065061047,
      "grad_norm": 4.7695469312968915e-09,
      "learning_rate": 8.142124418034385e-08,
      "logits/chosen": 9.141852378845215,
      "logits/rejected": 8.288122177124023,
      "logps/chosen": -43.673988342285156,
      "logps/rejected": -44.82506561279297,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.673988342285156,
      "rewards/margins": 1.151077389717102,
      "rewards/rejected": -44.82506561279297,
      "step": 4675
    },
    {
      "epoch": 2.5047666833918716,
      "grad_norm": 1.980043802572538e-10,
      "learning_rate": 8.057143619167073e-08,
      "logits/chosen": 8.759631156921387,
      "logits/rejected": 8.601361274719238,
      "logps/chosen": -43.098846435546875,
      "logps/rejected": -44.34484100341797,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -43.098846435546875,
      "rewards/margins": 1.2459893226623535,
      "rewards/rejected": -44.34484100341797,
      "step": 4680
    },
    {
      "epoch": 2.507442716173273,
      "grad_norm": 2.619037474401766e-10,
      "learning_rate": 7.97256975007633e-08,
      "logits/chosen": 9.121816635131836,
      "logits/rejected": 8.402771949768066,
      "logps/chosen": -42.12567138671875,
      "logps/rejected": -43.63753890991211,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -42.12567138671875,
      "rewards/margins": 1.5118703842163086,
      "rewards/rejected": -43.63753890991211,
      "step": 4685
    },
    {
      "epoch": 2.5101187489546746,
      "grad_norm": 0.09768188870803572,
      "learning_rate": 7.888403631298186e-08,
      "logits/chosen": 8.689119338989258,
      "logits/rejected": 8.800952911376953,
      "logps/chosen": -42.88166427612305,
      "logps/rejected": -44.109832763671875,
      "loss": 0.0002,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -42.88166427612305,
      "rewards/margins": 1.228164792060852,
      "rewards/rejected": -44.109832763671875,
      "step": 4690
    },
    {
      "epoch": 2.5127947817360763,
      "grad_norm": 4.538492359074244e-09,
      "learning_rate": 7.804646079412719e-08,
      "logits/chosen": 8.839693069458008,
      "logits/rejected": 8.37195110321045,
      "logps/chosen": -42.27249526977539,
      "logps/rejected": -43.85920715332031,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.27249526977539,
      "rewards/margins": 1.5867151021957397,
      "rewards/rejected": -43.85920715332031,
      "step": 4695
    },
    {
      "epoch": 2.515470814517478,
      "grad_norm": 2.7601590985472095e-10,
      "learning_rate": 7.72129790703604e-08,
      "logits/chosen": 9.232236862182617,
      "logits/rejected": 8.781030654907227,
      "logps/chosen": -42.43456268310547,
      "logps/rejected": -43.71471405029297,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.43456268310547,
      "rewards/margins": 1.280151128768921,
      "rewards/rejected": -43.71471405029297,
      "step": 4700
    },
    {
      "epoch": 2.5181468472988793,
      "grad_norm": 3.567319141993614e-05,
      "learning_rate": 7.638359922812504e-08,
      "logits/chosen": 8.958989143371582,
      "logits/rejected": 8.665552139282227,
      "logps/chosen": -43.41173553466797,
      "logps/rejected": -43.88212585449219,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -43.41173553466797,
      "rewards/margins": 0.47039252519607544,
      "rewards/rejected": -43.88212585449219,
      "step": 4705
    },
    {
      "epoch": 2.520822880080281,
      "grad_norm": 1.4408883132354555e-07,
      "learning_rate": 7.555832931406774e-08,
      "logits/chosen": 9.118542671203613,
      "logits/rejected": 8.412516593933105,
      "logps/chosen": -42.43720245361328,
      "logps/rejected": -44.114192962646484,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -42.43720245361328,
      "rewards/margins": 1.6769895553588867,
      "rewards/rejected": -44.114192962646484,
      "step": 4710
    },
    {
      "epoch": 2.5234989128616827,
      "grad_norm": 6.101686581962118e-13,
      "learning_rate": 7.47371773349611e-08,
      "logits/chosen": 8.923929214477539,
      "logits/rejected": 8.91518783569336,
      "logps/chosen": -42.96770095825195,
      "logps/rejected": -44.249420166015625,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -42.96770095825195,
      "rewards/margins": 1.2817184925079346,
      "rewards/rejected": -44.249420166015625,
      "step": 4715
    },
    {
      "epoch": 2.526174945643084,
      "grad_norm": 6.473310034087893e-06,
      "learning_rate": 7.392015125762496e-08,
      "logits/chosen": 8.949080467224121,
      "logits/rejected": 8.5923490524292,
      "logps/chosen": -43.18629837036133,
      "logps/rejected": -44.24211883544922,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.18629837036133,
      "rewards/margins": 1.0558218955993652,
      "rewards/rejected": -44.24211883544922,
      "step": 4720
    },
    {
      "epoch": 2.5288509784244857,
      "grad_norm": 7.948510177439446e-07,
      "learning_rate": 7.310725900885018e-08,
      "logits/chosen": 9.120261192321777,
      "logits/rejected": 9.004095077514648,
      "logps/chosen": -43.56443786621094,
      "logps/rejected": -43.506187438964844,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -43.56443786621094,
      "rewards/margins": -0.05825188010931015,
      "rewards/rejected": -43.506187438964844,
      "step": 4725
    },
    {
      "epoch": 2.5315270112058874,
      "grad_norm": 9.222741127483986e-10,
      "learning_rate": 7.229850847532076e-08,
      "logits/chosen": 8.934659004211426,
      "logits/rejected": 8.441221237182617,
      "logps/chosen": -43.583702087402344,
      "logps/rejected": -44.898075103759766,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -43.583702087402344,
      "rewards/margins": 1.314375877380371,
      "rewards/rejected": -44.898075103759766,
      "step": 4730
    },
    {
      "epoch": 2.5342030439872887,
      "grad_norm": 4.4504388257674365e-09,
      "learning_rate": 7.149390750353779e-08,
      "logits/chosen": 8.475000381469727,
      "logits/rejected": 8.893838882446289,
      "logps/chosen": -42.61485290527344,
      "logps/rejected": -43.29773712158203,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.61485290527344,
      "rewards/margins": 0.6828866004943848,
      "rewards/rejected": -43.29773712158203,
      "step": 4735
    },
    {
      "epoch": 2.5368790767686904,
      "grad_norm": 2.551103097822002e-11,
      "learning_rate": 7.069346389974374e-08,
      "logits/chosen": 9.223368644714355,
      "logits/rejected": 8.584945678710938,
      "logps/chosen": -42.723670959472656,
      "logps/rejected": -44.25645446777344,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -42.723670959472656,
      "rewards/margins": 1.5327866077423096,
      "rewards/rejected": -44.25645446777344,
      "step": 4740
    },
    {
      "epoch": 2.539555109550092,
      "grad_norm": 1.799314031257049e-08,
      "learning_rate": 6.989718542984563e-08,
      "logits/chosen": 9.01329517364502,
      "logits/rejected": 9.00218391418457,
      "logps/chosen": -43.31816864013672,
      "logps/rejected": -44.022430419921875,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.31816864013672,
      "rewards/margins": 0.7042635679244995,
      "rewards/rejected": -44.022430419921875,
      "step": 4745
    },
    {
      "epoch": 2.5422311423314934,
      "grad_norm": 2.013353981105687e-11,
      "learning_rate": 6.9105079819341e-08,
      "logits/chosen": 9.109373092651367,
      "logits/rejected": 7.922883033752441,
      "logps/chosen": -43.286582946777344,
      "logps/rejected": -44.85076904296875,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.286582946777344,
      "rewards/margins": 1.5641835927963257,
      "rewards/rejected": -44.85076904296875,
      "step": 4750
    },
    {
      "epoch": 2.544907175112895,
      "grad_norm": 4.363193649779592e-10,
      "learning_rate": 6.831715475324163e-08,
      "logits/chosen": 9.389158248901367,
      "logits/rejected": 8.734638214111328,
      "logps/chosen": -43.528594970703125,
      "logps/rejected": -44.85797882080078,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.528594970703125,
      "rewards/margins": 1.3293819427490234,
      "rewards/rejected": -44.85797882080078,
      "step": 4755
    },
    {
      "epoch": 2.547583207894297,
      "grad_norm": 6.73110666216195e-08,
      "learning_rate": 6.753341787600026e-08,
      "logits/chosen": 9.577058792114258,
      "logits/rejected": 9.203988075256348,
      "logps/chosen": -43.840457916259766,
      "logps/rejected": -44.656455993652344,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.840457916259766,
      "rewards/margins": 0.8160001635551453,
      "rewards/rejected": -44.656455993652344,
      "step": 4760
    },
    {
      "epoch": 2.5502592406756985,
      "grad_norm": 8.597062998286907e-11,
      "learning_rate": 6.67538767914353e-08,
      "logits/chosen": 9.302873611450195,
      "logits/rejected": 8.718219757080078,
      "logps/chosen": -43.999595642089844,
      "logps/rejected": -45.015052795410156,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.999595642089844,
      "rewards/margins": 1.0154579877853394,
      "rewards/rejected": -45.015052795410156,
      "step": 4765
    },
    {
      "epoch": 2.5529352734571,
      "grad_norm": 8.902378285635942e-06,
      "learning_rate": 6.597853906265793e-08,
      "logits/chosen": 8.784675598144531,
      "logits/rejected": 8.37619686126709,
      "logps/chosen": -42.801353454589844,
      "logps/rejected": -44.62187576293945,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -42.801353454589844,
      "rewards/margins": 1.8205153942108154,
      "rewards/rejected": -44.62187576293945,
      "step": 4770
    },
    {
      "epoch": 2.5556113062385015,
      "grad_norm": 1.6874402235025962e-11,
      "learning_rate": 6.5207412211998e-08,
      "logits/chosen": 8.940374374389648,
      "logits/rejected": 8.66877269744873,
      "logps/chosen": -43.24928665161133,
      "logps/rejected": -44.148502349853516,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -43.24928665161133,
      "rewards/margins": 0.8992165327072144,
      "rewards/rejected": -44.148502349853516,
      "step": 4775
    },
    {
      "epoch": 2.558287339019903,
      "grad_norm": 4.947176296804689e-07,
      "learning_rate": 6.444050372093186e-08,
      "logits/chosen": 9.13990592956543,
      "logits/rejected": 8.96580696105957,
      "logps/chosen": -43.406429290771484,
      "logps/rejected": -44.29347229003906,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.406429290771484,
      "rewards/margins": 0.8870447278022766,
      "rewards/rejected": -44.29347229003906,
      "step": 4780
    },
    {
      "epoch": 2.5609633718013045,
      "grad_norm": 2.61470305220332e-10,
      "learning_rate": 6.367782103000873e-08,
      "logits/chosen": 9.231401443481445,
      "logits/rejected": 8.809134483337402,
      "logps/chosen": -43.69819641113281,
      "logps/rejected": -44.2975959777832,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -43.69819641113281,
      "rewards/margins": 0.5994040966033936,
      "rewards/rejected": -44.2975959777832,
      "step": 4785
    },
    {
      "epoch": 2.5636394045827062,
      "grad_norm": 3.206934296860541e-11,
      "learning_rate": 6.29193715387798e-08,
      "logits/chosen": 8.969747543334961,
      "logits/rejected": 8.523025512695312,
      "logps/chosen": -42.60356903076172,
      "logps/rejected": -43.568359375,
      "loss": 0.0005,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.60356903076172,
      "rewards/margins": 0.9647868275642395,
      "rewards/rejected": -43.568359375,
      "step": 4790
    },
    {
      "epoch": 2.566315437364108,
      "grad_norm": 1.7486743581350295e-12,
      "learning_rate": 6.216516260572502e-08,
      "logits/chosen": 9.140700340270996,
      "logits/rejected": 8.706997871398926,
      "logps/chosen": -43.59941864013672,
      "logps/rejected": -44.77802276611328,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.59941864013672,
      "rewards/margins": 1.178605556488037,
      "rewards/rejected": -44.77802276611328,
      "step": 4795
    },
    {
      "epoch": 2.568991470145509,
      "grad_norm": 1.05459567632616e-08,
      "learning_rate": 6.141520154818297e-08,
      "logits/chosen": 9.3609037399292,
      "logits/rejected": 8.919872283935547,
      "logps/chosen": -43.40058898925781,
      "logps/rejected": -44.294105529785156,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.40058898925781,
      "rewards/margins": 0.8935127258300781,
      "rewards/rejected": -44.294105529785156,
      "step": 4800
    },
    {
      "epoch": 2.568991470145509,
      "eval_logits/chosen": 7.669034481048584,
      "eval_logits/rejected": 7.379368305206299,
      "eval_logps/chosen": -43.0767936706543,
      "eval_logps/rejected": -44.17669677734375,
      "eval_loss": 2.33503300250959e-07,
      "eval_rewards/accuracies": 0.5675074458122253,
      "eval_rewards/chosen": -43.0767936706543,
      "eval_rewards/margins": 1.099900245666504,
      "eval_rewards/rejected": -44.17669677734375,
      "eval_runtime": 40.4965,
      "eval_samples_per_second": 33.213,
      "eval_steps_per_second": 8.322,
      "step": 4800
    },
    {
      "epoch": 2.571667502926911,
      "grad_norm": 1.3818118511475883e-05,
      "learning_rate": 6.066949564227897e-08,
      "logits/chosen": 9.336973190307617,
      "logits/rejected": 8.907478332519531,
      "logps/chosen": -42.392398834228516,
      "logps/rejected": -43.604225158691406,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.392398834228516,
      "rewards/margins": 1.2118293046951294,
      "rewards/rejected": -43.604225158691406,
      "step": 4805
    },
    {
      "epoch": 2.574343535708312,
      "grad_norm": 7.631184559034695e-07,
      "learning_rate": 5.992805212285523e-08,
      "logits/chosen": 9.003918647766113,
      "logits/rejected": 8.74857234954834,
      "logps/chosen": -42.74648666381836,
      "logps/rejected": -44.06636428833008,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.74648666381836,
      "rewards/margins": 1.3198761940002441,
      "rewards/rejected": -44.06636428833008,
      "step": 4810
    },
    {
      "epoch": 2.577019568489714,
      "grad_norm": 8.208916666334157e-09,
      "learning_rate": 5.9190878183399684e-08,
      "logits/chosen": 9.21426010131836,
      "logits/rejected": 8.546037673950195,
      "logps/chosen": -43.744224548339844,
      "logps/rejected": -45.308162689208984,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -43.744224548339844,
      "rewards/margins": 1.563937783241272,
      "rewards/rejected": -45.308162689208984,
      "step": 4815
    },
    {
      "epoch": 2.5796956012711156,
      "grad_norm": 5.66179956353687e-07,
      "learning_rate": 5.845798097597748e-08,
      "logits/chosen": 8.841906547546387,
      "logits/rejected": 8.718778610229492,
      "logps/chosen": -43.10283660888672,
      "logps/rejected": -44.05834197998047,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.10283660888672,
      "rewards/margins": 0.9555104970932007,
      "rewards/rejected": -44.05834197998047,
      "step": 4820
    },
    {
      "epoch": 2.5823716340525174,
      "grad_norm": 2.7991797546243745e-09,
      "learning_rate": 5.772936761116026e-08,
      "logits/chosen": 9.157719612121582,
      "logits/rejected": 8.6575288772583,
      "logps/chosen": -43.11156463623047,
      "logps/rejected": -44.14491653442383,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.11156463623047,
      "rewards/margins": 1.0333516597747803,
      "rewards/rejected": -44.14491653442383,
      "step": 4825
    },
    {
      "epoch": 2.5850476668339186,
      "grad_norm": 8.051157589167438e-12,
      "learning_rate": 5.700504515795829e-08,
      "logits/chosen": 9.240645408630371,
      "logits/rejected": 8.733817100524902,
      "logps/chosen": -42.72593307495117,
      "logps/rejected": -44.708248138427734,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.72593307495117,
      "rewards/margins": 1.9823105335235596,
      "rewards/rejected": -44.708248138427734,
      "step": 4830
    },
    {
      "epoch": 2.5877236996153203,
      "grad_norm": 2.1256045419031086e-12,
      "learning_rate": 5.628502064375101e-08,
      "logits/chosen": 9.201604843139648,
      "logits/rejected": 8.701864242553711,
      "logps/chosen": -43.109962463378906,
      "logps/rejected": -45.54670333862305,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -43.109962463378906,
      "rewards/margins": 2.436739683151245,
      "rewards/rejected": -45.54670333862305,
      "step": 4835
    },
    {
      "epoch": 2.5903997323967216,
      "grad_norm": 2.305228283750965e-09,
      "learning_rate": 5.55693010542197e-08,
      "logits/chosen": 9.403355598449707,
      "logits/rejected": 8.471258163452148,
      "logps/chosen": -43.800025939941406,
      "logps/rejected": -44.85037612915039,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.800025939941406,
      "rewards/margins": 1.0503461360931396,
      "rewards/rejected": -44.85037612915039,
      "step": 4840
    },
    {
      "epoch": 2.5930757651781233,
      "grad_norm": 5.82498911742165e-14,
      "learning_rate": 5.485789333327856e-08,
      "logits/chosen": 9.017709732055664,
      "logits/rejected": 8.868873596191406,
      "logps/chosen": -43.83453369140625,
      "logps/rejected": -45.265838623046875,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.83453369140625,
      "rewards/margins": 1.4313052892684937,
      "rewards/rejected": -45.265838623046875,
      "step": 4845
    },
    {
      "epoch": 2.595751797959525,
      "grad_norm": 4.770058159357817e-09,
      "learning_rate": 5.4150804383008675e-08,
      "logits/chosen": 9.21560001373291,
      "logits/rejected": 8.71939468383789,
      "logps/chosen": -42.4356689453125,
      "logps/rejected": -44.408084869384766,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -42.4356689453125,
      "rewards/margins": 1.9724204540252686,
      "rewards/rejected": -44.408084869384766,
      "step": 4850
    },
    {
      "epoch": 2.5984278307409268,
      "grad_norm": 1.6432633084823913e-06,
      "learning_rate": 5.344804106359002e-08,
      "logits/chosen": 9.005030632019043,
      "logits/rejected": 8.43646240234375,
      "logps/chosen": -43.049781799316406,
      "logps/rejected": -44.541805267333984,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -43.049781799316406,
      "rewards/margins": 1.492021083831787,
      "rewards/rejected": -44.541805267333984,
      "step": 4855
    },
    {
      "epoch": 2.601103863522328,
      "grad_norm": 0.00014917573958102027,
      "learning_rate": 5.274961019323559e-08,
      "logits/chosen": 8.91691780090332,
      "logits/rejected": 8.580900192260742,
      "logps/chosen": -42.616825103759766,
      "logps/rejected": -44.42259216308594,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.616825103759766,
      "rewards/margins": 1.8057644367218018,
      "rewards/rejected": -44.42259216308594,
      "step": 4860
    },
    {
      "epoch": 2.6037798963037297,
      "grad_norm": 3.087445973846156e-05,
      "learning_rate": 5.205551854812451e-08,
      "logits/chosen": 9.534845352172852,
      "logits/rejected": 9.058837890625,
      "logps/chosen": -42.86024856567383,
      "logps/rejected": -43.009254455566406,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -42.86024856567383,
      "rewards/margins": 0.1490052044391632,
      "rewards/rejected": -43.009254455566406,
      "step": 4865
    },
    {
      "epoch": 2.606455929085131,
      "grad_norm": 3.996685825953934e-11,
      "learning_rate": 5.1365772862337177e-08,
      "logits/chosen": 8.968287467956543,
      "logits/rejected": 8.726919174194336,
      "logps/chosen": -43.56601333618164,
      "logps/rejected": -45.56330108642578,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.56601333618164,
      "rewards/margins": 1.997291922569275,
      "rewards/rejected": -45.56330108642578,
      "step": 4870
    },
    {
      "epoch": 2.6091319618665327,
      "grad_norm": 2.458635185533661e-05,
      "learning_rate": 5.068037982778905e-08,
      "logits/chosen": 8.992135047912598,
      "logits/rejected": 8.74323844909668,
      "logps/chosen": -43.59288024902344,
      "logps/rejected": -44.756614685058594,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.59288024902344,
      "rewards/margins": 1.163736343383789,
      "rewards/rejected": -44.756614685058594,
      "step": 4875
    },
    {
      "epoch": 2.6118079946479344,
      "grad_norm": 3.509076287800465e-11,
      "learning_rate": 4.999934609416656e-08,
      "logits/chosen": 9.304932594299316,
      "logits/rejected": 8.768881797790527,
      "logps/chosen": -43.59864044189453,
      "logps/rejected": -45.254886627197266,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.59864044189453,
      "rewards/margins": 1.656252145767212,
      "rewards/rejected": -45.254886627197266,
      "step": 4880
    },
    {
      "epoch": 2.614484027429336,
      "grad_norm": 1.284089324467653e-06,
      "learning_rate": 4.932267826886183e-08,
      "logits/chosen": 8.986671447753906,
      "logits/rejected": 8.93264102935791,
      "logps/chosen": -42.908203125,
      "logps/rejected": -43.92797088623047,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.908203125,
      "rewards/margins": 1.019768476486206,
      "rewards/rejected": -43.92797088623047,
      "step": 4885
    },
    {
      "epoch": 2.6171600602107374,
      "grad_norm": 7.56573416378216e-12,
      "learning_rate": 4.8650382916909206e-08,
      "logits/chosen": 9.429896354675293,
      "logits/rejected": 8.764261245727539,
      "logps/chosen": -43.6498908996582,
      "logps/rejected": -44.962730407714844,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.6498908996582,
      "rewards/margins": 1.3128423690795898,
      "rewards/rejected": -44.962730407714844,
      "step": 4890
    },
    {
      "epoch": 2.619836092992139,
      "grad_norm": 0.1268813660563405,
      "learning_rate": 4.7982466560920976e-08,
      "logits/chosen": 8.970413208007812,
      "logits/rejected": 8.704388618469238,
      "logps/chosen": -43.230384826660156,
      "logps/rejected": -42.64691925048828,
      "loss": 0.0009,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.230384826660156,
      "rewards/margins": -0.5834633111953735,
      "rewards/rejected": -42.64691925048828,
      "step": 4895
    },
    {
      "epoch": 2.622512125773541,
      "grad_norm": 3.930897497825209e-05,
      "learning_rate": 4.7318935681024685e-08,
      "logits/chosen": 8.904264450073242,
      "logits/rejected": 8.47325611114502,
      "logps/chosen": -42.75983810424805,
      "logps/rejected": -43.8365478515625,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.75983810424805,
      "rewards/margins": 1.0767098665237427,
      "rewards/rejected": -43.8365478515625,
      "step": 4900
    },
    {
      "epoch": 2.625188158554942,
      "grad_norm": 2.3387832181807664e-06,
      "learning_rate": 4.6659796714799745e-08,
      "logits/chosen": 8.630779266357422,
      "logits/rejected": 8.169549942016602,
      "logps/chosen": -42.07412338256836,
      "logps/rejected": -43.108097076416016,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.07412338256836,
      "rewards/margins": 1.0339674949645996,
      "rewards/rejected": -43.108097076416016,
      "step": 4905
    },
    {
      "epoch": 2.627864191336344,
      "grad_norm": 1.1770521225739604e-09,
      "learning_rate": 4.60050560572155e-08,
      "logits/chosen": 9.008570671081543,
      "logits/rejected": 9.138814926147461,
      "logps/chosen": -44.2877082824707,
      "logps/rejected": -44.840511322021484,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -44.2877082824707,
      "rewards/margins": 0.552801251411438,
      "rewards/rejected": -44.840511322021484,
      "step": 4910
    },
    {
      "epoch": 2.6305402241177456,
      "grad_norm": 3.6869112562877167e-10,
      "learning_rate": 4.535472006056834e-08,
      "logits/chosen": 9.11001968383789,
      "logits/rejected": 8.620874404907227,
      "logps/chosen": -43.35248565673828,
      "logps/rejected": -44.16017150878906,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.35248565673828,
      "rewards/margins": 0.8076847195625305,
      "rewards/rejected": -44.16017150878906,
      "step": 4915
    },
    {
      "epoch": 2.6332162568991473,
      "grad_norm": 5.560416415014575e-13,
      "learning_rate": 4.470879503442132e-08,
      "logits/chosen": 8.919614791870117,
      "logits/rejected": 8.703848838806152,
      "logps/chosen": -43.5286865234375,
      "logps/rejected": -44.891990661621094,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -43.5286865234375,
      "rewards/margins": 1.363307237625122,
      "rewards/rejected": -44.891990661621094,
      "step": 4920
    },
    {
      "epoch": 2.6358922896805486,
      "grad_norm": 4.5325250170831703e-13,
      "learning_rate": 4.406728724554154e-08,
      "logits/chosen": 9.218072891235352,
      "logits/rejected": 8.42087173461914,
      "logps/chosen": -42.3083381652832,
      "logps/rejected": -44.92945098876953,
      "loss": 0.0,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -42.3083381652832,
      "rewards/margins": 2.621107578277588,
      "rewards/rejected": -44.92945098876953,
      "step": 4925
    },
    {
      "epoch": 2.6385683224619503,
      "grad_norm": 4.8884796417525246e-05,
      "learning_rate": 4.3430202917840664e-08,
      "logits/chosen": 9.42890739440918,
      "logits/rejected": 8.808436393737793,
      "logps/chosen": -44.56859588623047,
      "logps/rejected": -46.297515869140625,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -44.56859588623047,
      "rewards/margins": 1.728915810585022,
      "rewards/rejected": -46.297515869140625,
      "step": 4930
    },
    {
      "epoch": 2.6412443552433515,
      "grad_norm": 4.395523534731467e-07,
      "learning_rate": 4.279754823231346e-08,
      "logits/chosen": 9.074231147766113,
      "logits/rejected": 8.443303108215332,
      "logps/chosen": -41.81409454345703,
      "logps/rejected": -43.53511428833008,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.81409454345703,
      "rewards/margins": 1.7210206985473633,
      "rewards/rejected": -43.53511428833008,
      "step": 4935
    },
    {
      "epoch": 2.6439203880247533,
      "grad_norm": 4.885682021598219e-12,
      "learning_rate": 4.216932932697859e-08,
      "logits/chosen": 9.230489730834961,
      "logits/rejected": 8.875797271728516,
      "logps/chosen": -43.60871124267578,
      "logps/rejected": -45.03398895263672,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.60871124267578,
      "rewards/margins": 1.4252796173095703,
      "rewards/rejected": -45.03398895263672,
      "step": 4940
    },
    {
      "epoch": 2.646596420806155,
      "grad_norm": 3.7150421522036104e-08,
      "learning_rate": 4.154555229681844e-08,
      "logits/chosen": 8.929789543151855,
      "logits/rejected": 8.41897201538086,
      "logps/chosen": -43.34040451049805,
      "logps/rejected": -44.39496612548828,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.34040451049805,
      "rewards/margins": 1.0545607805252075,
      "rewards/rejected": -44.39496612548828,
      "step": 4945
    },
    {
      "epoch": 2.6492724535875567,
      "grad_norm": 3.8184685541067486e-09,
      "learning_rate": 4.092622319372069e-08,
      "logits/chosen": 9.147625923156738,
      "logits/rejected": 8.522071838378906,
      "logps/chosen": -43.4959602355957,
      "logps/rejected": -44.520790100097656,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.4959602355957,
      "rewards/margins": 1.024835467338562,
      "rewards/rejected": -44.520790100097656,
      "step": 4950
    },
    {
      "epoch": 2.651948486368958,
      "grad_norm": 6.386522240649272e-08,
      "learning_rate": 4.031134802641889e-08,
      "logits/chosen": 9.138346672058105,
      "logits/rejected": 9.06105899810791,
      "logps/chosen": -43.88892364501953,
      "logps/rejected": -43.99091720581055,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.88892364501953,
      "rewards/margins": 0.1019887924194336,
      "rewards/rejected": -43.99091720581055,
      "step": 4955
    },
    {
      "epoch": 2.6546245191503597,
      "grad_norm": 4.3663886471168935e-11,
      "learning_rate": 3.970093276043468e-08,
      "logits/chosen": 9.051881790161133,
      "logits/rejected": 8.854409217834473,
      "logps/chosen": -43.49562454223633,
      "logps/rejected": -44.75468826293945,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.49562454223633,
      "rewards/margins": 1.2590659856796265,
      "rewards/rejected": -44.75468826293945,
      "step": 4960
    },
    {
      "epoch": 2.657300551931761,
      "grad_norm": 2.6079622630085887e-08,
      "learning_rate": 3.9094983318019584e-08,
      "logits/chosen": 9.301385879516602,
      "logits/rejected": 8.678624153137207,
      "logps/chosen": -43.20638656616211,
      "logps/rejected": -44.19391632080078,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.20638656616211,
      "rewards/margins": 0.9875224232673645,
      "rewards/rejected": -44.19391632080078,
      "step": 4965
    },
    {
      "epoch": 2.6599765847131627,
      "grad_norm": 3.0649162510362826e-09,
      "learning_rate": 3.849350557809789e-08,
      "logits/chosen": 9.169889450073242,
      "logits/rejected": 9.004755973815918,
      "logps/chosen": -43.71628189086914,
      "logps/rejected": -44.48175048828125,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.71628189086914,
      "rewards/margins": 0.7654660940170288,
      "rewards/rejected": -44.48175048828125,
      "step": 4970
    },
    {
      "epoch": 2.6626526174945644,
      "grad_norm": 1.2666338350319802e-12,
      "learning_rate": 3.789650537620903e-08,
      "logits/chosen": 9.083078384399414,
      "logits/rejected": 8.865434646606445,
      "logps/chosen": -44.71297836303711,
      "logps/rejected": -45.582454681396484,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -44.71297836303711,
      "rewards/margins": 0.8694734573364258,
      "rewards/rejected": -45.582454681396484,
      "step": 4975
    },
    {
      "epoch": 2.665328650275966,
      "grad_norm": 2.971531394870502e-07,
      "learning_rate": 3.730398850445182e-08,
      "logits/chosen": 8.873515129089355,
      "logits/rejected": 8.76361083984375,
      "logps/chosen": -43.24320602416992,
      "logps/rejected": -43.78880310058594,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.24320602416992,
      "rewards/margins": 0.545599639415741,
      "rewards/rejected": -43.78880310058594,
      "step": 4980
    },
    {
      "epoch": 2.6680046830573674,
      "grad_norm": 1.5357519756630864e-11,
      "learning_rate": 3.671596071142735e-08,
      "logits/chosen": 9.174957275390625,
      "logits/rejected": 8.634170532226562,
      "logps/chosen": -43.926204681396484,
      "logps/rejected": -45.98197937011719,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -43.926204681396484,
      "rewards/margins": 2.055774688720703,
      "rewards/rejected": -45.98197937011719,
      "step": 4985
    },
    {
      "epoch": 2.670680715838769,
      "grad_norm": 7.355422836845194e-09,
      "learning_rate": 3.6132427702183996e-08,
      "logits/chosen": 9.440366744995117,
      "logits/rejected": 8.885251998901367,
      "logps/chosen": -43.57441711425781,
      "logps/rejected": -44.79595184326172,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.57441711425781,
      "rewards/margins": 1.2215327024459839,
      "rewards/rejected": -44.79595184326172,
      "step": 4990
    },
    {
      "epoch": 2.6733567486201704,
      "grad_norm": 1.0379499255378706e-12,
      "learning_rate": 3.555339513816147e-08,
      "logits/chosen": 9.349838256835938,
      "logits/rejected": 9.28815746307373,
      "logps/chosen": -45.09845733642578,
      "logps/rejected": -44.304115295410156,
      "loss": 0.0,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -45.09845733642578,
      "rewards/margins": -0.7943447828292847,
      "rewards/rejected": -44.304115295410156,
      "step": 4995
    },
    {
      "epoch": 2.676032781401572,
      "grad_norm": 1.2105664935416367e-08,
      "learning_rate": 3.497886863713639e-08,
      "logits/chosen": 9.391852378845215,
      "logits/rejected": 9.19373893737793,
      "logps/chosen": -45.093528747558594,
      "logps/rejected": -44.81812286376953,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -45.093528747558594,
      "rewards/margins": -0.27539995312690735,
      "rewards/rejected": -44.81812286376953,
      "step": 5000
    },
    {
      "epoch": 2.678708814182974,
      "grad_norm": 4.991499452668781e-09,
      "learning_rate": 3.440885377316721e-08,
      "logits/chosen": 8.918176651000977,
      "logits/rejected": 8.692453384399414,
      "logps/chosen": -43.148006439208984,
      "logps/rejected": -43.84690475463867,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.148006439208984,
      "rewards/margins": 0.6988977789878845,
      "rewards/rejected": -43.84690475463867,
      "step": 5005
    },
    {
      "epoch": 2.6813848469643755,
      "grad_norm": 8.902621115785179e-10,
      "learning_rate": 3.384335607654082e-08,
      "logits/chosen": 8.754938125610352,
      "logits/rejected": 8.3749418258667,
      "logps/chosen": -43.56318283081055,
      "logps/rejected": -44.35714340209961,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.56318283081055,
      "rewards/margins": 0.7939629554748535,
      "rewards/rejected": -44.35714340209961,
      "step": 5010
    },
    {
      "epoch": 2.684060879745777,
      "grad_norm": 0.00010227740881515844,
      "learning_rate": 3.328238103371811e-08,
      "logits/chosen": 9.126741409301758,
      "logits/rejected": 8.923644065856934,
      "logps/chosen": -44.1834831237793,
      "logps/rejected": -44.94786834716797,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.1834831237793,
      "rewards/margins": 0.7643880248069763,
      "rewards/rejected": -44.94786834716797,
      "step": 5015
    },
    {
      "epoch": 2.6867369125271785,
      "grad_norm": 4.3713092844715224e-12,
      "learning_rate": 3.272593408728169e-08,
      "logits/chosen": 9.143709182739258,
      "logits/rejected": 8.276471138000488,
      "logps/chosen": -43.83470916748047,
      "logps/rejected": -45.55449676513672,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.83470916748047,
      "rewards/margins": 1.719788908958435,
      "rewards/rejected": -45.55449676513672,
      "step": 5020
    },
    {
      "epoch": 2.6894129453085798,
      "grad_norm": 1.689941290693259e-11,
      "learning_rate": 3.217402063588204e-08,
      "logits/chosen": 9.257261276245117,
      "logits/rejected": 8.785032272338867,
      "logps/chosen": -43.62569808959961,
      "logps/rejected": -44.92503356933594,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.62569808959961,
      "rewards/margins": 1.2993357181549072,
      "rewards/rejected": -44.92503356933594,
      "step": 5025
    },
    {
      "epoch": 2.6920889780899815,
      "grad_norm": 2.713641675516236e-10,
      "learning_rate": 3.162664603418608e-08,
      "logits/chosen": 9.411273956298828,
      "logits/rejected": 9.02724552154541,
      "logps/chosen": -44.0105094909668,
      "logps/rejected": -45.17673110961914,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.0105094909668,
      "rewards/margins": 1.1662204265594482,
      "rewards/rejected": -45.17673110961914,
      "step": 5030
    },
    {
      "epoch": 2.694765010871383,
      "grad_norm": 5.56789214960003e-10,
      "learning_rate": 3.1083815592824416e-08,
      "logits/chosen": 9.001391410827637,
      "logits/rejected": 8.826278686523438,
      "logps/chosen": -43.243263244628906,
      "logps/rejected": -44.79766082763672,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.243263244628906,
      "rewards/margins": 1.5543973445892334,
      "rewards/rejected": -44.79766082763672,
      "step": 5035
    },
    {
      "epoch": 2.697441043652785,
      "grad_norm": 2.1635836104737904e-08,
      "learning_rate": 3.054553457834053e-08,
      "logits/chosen": 8.783475875854492,
      "logits/rejected": 8.934747695922852,
      "logps/chosen": -42.744544982910156,
      "logps/rejected": -43.472389221191406,
      "loss": 0.0,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -42.744544982910156,
      "rewards/margins": 0.7278450727462769,
      "rewards/rejected": -43.472389221191406,
      "step": 5040
    },
    {
      "epoch": 2.700117076434186,
      "grad_norm": 1.2921655467812956e-07,
      "learning_rate": 3.0011808213139036e-08,
      "logits/chosen": 8.854515075683594,
      "logits/rejected": 9.011656761169434,
      "logps/chosen": -43.90321350097656,
      "logps/rejected": -44.64320373535156,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.90321350097656,
      "rewards/margins": 0.7399893999099731,
      "rewards/rejected": -44.64320373535156,
      "step": 5045
    },
    {
      "epoch": 2.702793109215588,
      "grad_norm": 1.7148536393523832e-08,
      "learning_rate": 2.948264167543568e-08,
      "logits/chosen": 9.019979476928711,
      "logits/rejected": 8.739091873168945,
      "logps/chosen": -44.38550567626953,
      "logps/rejected": -44.7628059387207,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.38550567626953,
      "rewards/margins": 0.377299964427948,
      "rewards/rejected": -44.7628059387207,
      "step": 5050
    },
    {
      "epoch": 2.7054691419969896,
      "grad_norm": 2.888795611051064e-10,
      "learning_rate": 2.8958040099206216e-08,
      "logits/chosen": 9.428553581237793,
      "logits/rejected": 9.128547668457031,
      "logps/chosen": -44.93156051635742,
      "logps/rejected": -45.50389099121094,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.93156051635742,
      "rewards/margins": 0.5723268389701843,
      "rewards/rejected": -45.50389099121094,
      "step": 5055
    },
    {
      "epoch": 2.708145174778391,
      "grad_norm": 5.1054413534218807e-11,
      "learning_rate": 2.843800857413775e-08,
      "logits/chosen": 8.756775856018066,
      "logits/rejected": 8.636309623718262,
      "logps/chosen": -43.08277130126953,
      "logps/rejected": -43.82593536376953,
      "loss": 0.0,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -43.08277130126953,
      "rewards/margins": 0.7431662678718567,
      "rewards/rejected": -43.82593536376953,
      "step": 5060
    },
    {
      "epoch": 2.7108212075597926,
      "grad_norm": 1.5235207796454951e-10,
      "learning_rate": 2.7922552145578203e-08,
      "logits/chosen": 9.130325317382812,
      "logits/rejected": 8.246222496032715,
      "logps/chosen": -43.436309814453125,
      "logps/rejected": -44.406898498535156,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.436309814453125,
      "rewards/margins": 0.9705861210823059,
      "rewards/rejected": -44.406898498535156,
      "step": 5065
    },
    {
      "epoch": 2.7134972403411943,
      "grad_norm": 3.407476836458239e-10,
      "learning_rate": 2.7411675814488277e-08,
      "logits/chosen": 8.94955062866211,
      "logits/rejected": 8.54890251159668,
      "logps/chosen": -43.280242919921875,
      "logps/rejected": -43.54806900024414,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.280242919921875,
      "rewards/margins": 0.26782605051994324,
      "rewards/rejected": -43.54806900024414,
      "step": 5070
    },
    {
      "epoch": 2.7161732731225956,
      "grad_norm": 3.947587094236316e-06,
      "learning_rate": 2.690538453739216e-08,
      "logits/chosen": 9.172134399414062,
      "logits/rejected": 8.903919219970703,
      "logps/chosen": -42.591148376464844,
      "logps/rejected": -43.65777587890625,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -42.591148376464844,
      "rewards/margins": 1.066626787185669,
      "rewards/rejected": -43.65777587890625,
      "step": 5075
    },
    {
      "epoch": 2.7188493059039973,
      "grad_norm": 1.6267342835275215e-09,
      "learning_rate": 2.6403683226330298e-08,
      "logits/chosen": 9.345294952392578,
      "logits/rejected": 8.793591499328613,
      "logps/chosen": -43.54670715332031,
      "logps/rejected": -45.21221160888672,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -43.54670715332031,
      "rewards/margins": 1.6655025482177734,
      "rewards/rejected": -45.21221160888672,
      "step": 5080
    },
    {
      "epoch": 2.721525338685399,
      "grad_norm": 7.970079157218425e-11,
      "learning_rate": 2.5906576748810804e-08,
      "logits/chosen": 9.353070259094238,
      "logits/rejected": 8.84656047821045,
      "logps/chosen": -43.939056396484375,
      "logps/rejected": -45.37128829956055,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -43.939056396484375,
      "rewards/margins": 1.4322341680526733,
      "rewards/rejected": -45.37128829956055,
      "step": 5085
    },
    {
      "epoch": 2.7242013714668003,
      "grad_norm": 2.1930341711764064e-05,
      "learning_rate": 2.5414069927763016e-08,
      "logits/chosen": 9.542497634887695,
      "logits/rejected": 8.837213516235352,
      "logps/chosen": -42.942108154296875,
      "logps/rejected": -43.854305267333984,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.942108154296875,
      "rewards/margins": 0.9121963381767273,
      "rewards/rejected": -43.854305267333984,
      "step": 5090
    },
    {
      "epoch": 2.726877404248202,
      "grad_norm": 2.0450888032888466e-09,
      "learning_rate": 2.4926167541490185e-08,
      "logits/chosen": 9.348544120788574,
      "logits/rejected": 8.513127326965332,
      "logps/chosen": -42.83620071411133,
      "logps/rejected": -44.66901397705078,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -42.83620071411133,
      "rewards/margins": 1.8328173160552979,
      "rewards/rejected": -44.66901397705078,
      "step": 5095
    },
    {
      "epoch": 2.7295534370296037,
      "grad_norm": 6.433083452140862e-10,
      "learning_rate": 2.4442874323623574e-08,
      "logits/chosen": 9.062211036682129,
      "logits/rejected": 8.49016284942627,
      "logps/chosen": -42.86086654663086,
      "logps/rejected": -44.317771911621094,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.86086654663086,
      "rewards/margins": 1.456904649734497,
      "rewards/rejected": -44.317771911621094,
      "step": 5100
    },
    {
      "epoch": 2.7322294698110055,
      "grad_norm": 3.2046151819761444e-08,
      "learning_rate": 2.396419496307589e-08,
      "logits/chosen": 8.877019882202148,
      "logits/rejected": 8.395615577697754,
      "logps/chosen": -42.771446228027344,
      "logps/rejected": -44.42991256713867,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -42.771446228027344,
      "rewards/margins": 1.6584646701812744,
      "rewards/rejected": -44.42991256713867,
      "step": 5105
    },
    {
      "epoch": 2.7349055025924067,
      "grad_norm": 1.4199026468186926e-13,
      "learning_rate": 2.349013410399653e-08,
      "logits/chosen": 8.8574857711792,
      "logits/rejected": 8.671666145324707,
      "logps/chosen": -43.167930603027344,
      "logps/rejected": -44.862247467041016,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -43.167930603027344,
      "rewards/margins": 1.694319486618042,
      "rewards/rejected": -44.862247467041016,
      "step": 5110
    },
    {
      "epoch": 2.7375815353738084,
      "grad_norm": 2.279877509417109e-11,
      "learning_rate": 2.3020696345725954e-08,
      "logits/chosen": 9.263761520385742,
      "logits/rejected": 8.411904335021973,
      "logps/chosen": -42.049312591552734,
      "logps/rejected": -43.55626678466797,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -42.049312591552734,
      "rewards/margins": 1.506955862045288,
      "rewards/rejected": -43.55626678466797,
      "step": 5115
    },
    {
      "epoch": 2.7402575681552097,
      "grad_norm": 4.846755253120891e-06,
      "learning_rate": 2.2555886242751398e-08,
      "logits/chosen": 9.307716369628906,
      "logits/rejected": 9.046968460083008,
      "logps/chosen": -42.930198669433594,
      "logps/rejected": -43.85496520996094,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.930198669433594,
      "rewards/margins": 0.924761176109314,
      "rewards/rejected": -43.85496520996094,
      "step": 5120
    },
    {
      "epoch": 2.7429336009366114,
      "grad_norm": 1.00782934623768e-09,
      "learning_rate": 2.2095708304662453e-08,
      "logits/chosen": 9.613089561462402,
      "logits/rejected": 8.96796703338623,
      "logps/chosen": -44.13003921508789,
      "logps/rejected": -45.09586715698242,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.13003921508789,
      "rewards/margins": 0.9658284187316895,
      "rewards/rejected": -45.09586715698242,
      "step": 5125
    },
    {
      "epoch": 2.745609633718013,
      "grad_norm": 2.345035529209351e-07,
      "learning_rate": 2.16401669961076e-08,
      "logits/chosen": 9.60748291015625,
      "logits/rejected": 8.8858003616333,
      "logps/chosen": -42.96638488769531,
      "logps/rejected": -44.450279235839844,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.96638488769531,
      "rewards/margins": 1.4838933944702148,
      "rewards/rejected": -44.450279235839844,
      "step": 5130
    },
    {
      "epoch": 2.748285666499415,
      "grad_norm": 4.0046950769199335e-09,
      "learning_rate": 2.1189266736750532e-08,
      "logits/chosen": 9.073997497558594,
      "logits/rejected": 8.833829879760742,
      "logps/chosen": -44.25225830078125,
      "logps/rejected": -44.91228485107422,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.25225830078125,
      "rewards/margins": 0.6600229144096375,
      "rewards/rejected": -44.91228485107422,
      "step": 5135
    },
    {
      "epoch": 2.750961699280816,
      "grad_norm": 2.025038714399654e-06,
      "learning_rate": 2.0743011901227623e-08,
      "logits/chosen": 9.187370300292969,
      "logits/rejected": 8.696545600891113,
      "logps/chosen": -41.95415496826172,
      "logps/rejected": -42.715797424316406,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.95415496826172,
      "rewards/margins": 0.7616451978683472,
      "rewards/rejected": -42.715797424316406,
      "step": 5140
    },
    {
      "epoch": 2.753637732062218,
      "grad_norm": 2.6195640357372744e-06,
      "learning_rate": 2.030140681910508e-08,
      "logits/chosen": 9.051790237426758,
      "logits/rejected": 8.603446960449219,
      "logps/chosen": -43.37526321411133,
      "logps/rejected": -44.78037643432617,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.37526321411133,
      "rewards/margins": 1.4051129817962646,
      "rewards/rejected": -44.78037643432617,
      "step": 5145
    },
    {
      "epoch": 2.756313764843619,
      "grad_norm": 1.105252712389415e-10,
      "learning_rate": 1.986445577483753e-08,
      "logits/chosen": 9.452394485473633,
      "logits/rejected": 8.909395217895508,
      "logps/chosen": -43.29167938232422,
      "logps/rejected": -43.958072662353516,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -43.29167938232422,
      "rewards/margins": 0.666393518447876,
      "rewards/rejected": -43.958072662353516,
      "step": 5150
    },
    {
      "epoch": 2.758989797625021,
      "grad_norm": 3.97753562415548e-12,
      "learning_rate": 1.9432163007725765e-08,
      "logits/chosen": 9.33098316192627,
      "logits/rejected": 9.138799667358398,
      "logps/chosen": -43.150264739990234,
      "logps/rejected": -44.80061721801758,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -43.150264739990234,
      "rewards/margins": 1.6503534317016602,
      "rewards/rejected": -44.80061721801758,
      "step": 5155
    },
    {
      "epoch": 2.7616658304064226,
      "grad_norm": 4.292591427401099e-06,
      "learning_rate": 1.9004532711876297e-08,
      "logits/chosen": 8.967911720275879,
      "logits/rejected": 9.11368465423584,
      "logps/chosen": -42.864418029785156,
      "logps/rejected": -43.31922149658203,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -42.864418029785156,
      "rewards/margins": 0.45479750633239746,
      "rewards/rejected": -43.31922149658203,
      "step": 5160
    },
    {
      "epoch": 2.7643418631878243,
      "grad_norm": 5.76518072920128e-07,
      "learning_rate": 1.8581569036159928e-08,
      "logits/chosen": 9.069539070129395,
      "logits/rejected": 8.464866638183594,
      "logps/chosen": -42.33643341064453,
      "logps/rejected": -44.70124816894531,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.33643341064453,
      "rewards/margins": 2.3648159503936768,
      "rewards/rejected": -44.70124816894531,
      "step": 5165
    },
    {
      "epoch": 2.7670178959692255,
      "grad_norm": 1.6939526449981929e-07,
      "learning_rate": 1.8163276084172285e-08,
      "logits/chosen": 9.361352920532227,
      "logits/rejected": 9.145605087280273,
      "logps/chosen": -42.27976989746094,
      "logps/rejected": -43.554710388183594,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -42.27976989746094,
      "rewards/margins": 1.2749345302581787,
      "rewards/rejected": -43.554710388183594,
      "step": 5170
    },
    {
      "epoch": 2.7696939287506273,
      "grad_norm": 3.3088187296791866e-06,
      "learning_rate": 1.7749657914193194e-08,
      "logits/chosen": 8.967318534851074,
      "logits/rejected": 8.859064102172852,
      "logps/chosen": -43.4511833190918,
      "logps/rejected": -44.335533142089844,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.4511833190918,
      "rewards/margins": 0.8843482732772827,
      "rewards/rejected": -44.335533142089844,
      "step": 5175
    },
    {
      "epoch": 2.7723699615320285,
      "grad_norm": 2.2041138659599504e-09,
      "learning_rate": 1.7340718539148203e-08,
      "logits/chosen": 9.051706314086914,
      "logits/rejected": 8.825541496276855,
      "logps/chosen": -43.35496139526367,
      "logps/rejected": -44.44926834106445,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.35496139526367,
      "rewards/margins": 1.094308614730835,
      "rewards/rejected": -44.44926834106445,
      "step": 5180
    },
    {
      "epoch": 2.7750459943134302,
      "grad_norm": 5.282325397021928e-11,
      "learning_rate": 1.6936461926568724e-08,
      "logits/chosen": 9.262999534606934,
      "logits/rejected": 8.572446823120117,
      "logps/chosen": -43.500022888183594,
      "logps/rejected": -45.655517578125,
      "loss": 0.0012,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.500022888183594,
      "rewards/margins": 2.1554973125457764,
      "rewards/rejected": -45.655517578125,
      "step": 5185
    },
    {
      "epoch": 2.777722027094832,
      "grad_norm": 5.0751135263184296e-11,
      "learning_rate": 1.6536891998554346e-08,
      "logits/chosen": 9.05966854095459,
      "logits/rejected": 8.62863540649414,
      "logps/chosen": -42.68769836425781,
      "logps/rejected": -44.32209777832031,
      "loss": 0.0001,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.68769836425781,
      "rewards/margins": 1.6343997716903687,
      "rewards/rejected": -44.32209777832031,
      "step": 5190
    },
    {
      "epoch": 2.7803980598762337,
      "grad_norm": 1.265507782643499e-07,
      "learning_rate": 1.6142012631734093e-08,
      "logits/chosen": 9.037896156311035,
      "logits/rejected": 8.6544189453125,
      "logps/chosen": -43.891319274902344,
      "logps/rejected": -44.3411750793457,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.891319274902344,
      "rewards/margins": 0.4498554766178131,
      "rewards/rejected": -44.3411750793457,
      "step": 5195
    },
    {
      "epoch": 2.783074092657635,
      "grad_norm": 3.1932820135416846e-11,
      "learning_rate": 1.575182765722949e-08,
      "logits/chosen": 9.161626815795898,
      "logits/rejected": 8.609975814819336,
      "logps/chosen": -43.45505905151367,
      "logps/rejected": -45.027748107910156,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.45505905151367,
      "rewards/margins": 1.5726863145828247,
      "rewards/rejected": -45.027748107910156,
      "step": 5200
    },
    {
      "epoch": 2.783074092657635,
      "eval_logits/chosen": 7.59325647354126,
      "eval_logits/rejected": 7.296000003814697,
      "eval_logps/chosen": -43.122711181640625,
      "eval_logps/rejected": -44.229122161865234,
      "eval_loss": 2.156311182943682e-07,
      "eval_rewards/accuracies": 0.5689911246299744,
      "eval_rewards/chosen": -43.122711181640625,
      "eval_rewards/margins": 1.106404423713684,
      "eval_rewards/rejected": -44.229122161865234,
      "eval_runtime": 40.4053,
      "eval_samples_per_second": 33.288,
      "eval_steps_per_second": 8.34,
      "step": 5200
    },
    {
      "epoch": 2.7857501254390367,
      "grad_norm": 2.3313276506519999e-07,
      "learning_rate": 1.536634086061672e-08,
      "logits/chosen": 8.959929466247559,
      "logits/rejected": 8.747356414794922,
      "logps/chosen": -42.357261657714844,
      "logps/rejected": -42.94976043701172,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.357261657714844,
      "rewards/margins": 0.5924930572509766,
      "rewards/rejected": -42.94976043701172,
      "step": 5205
    },
    {
      "epoch": 2.788426158220438,
      "grad_norm": 4.441210445552706e-08,
      "learning_rate": 1.4985555981890495e-08,
      "logits/chosen": 9.194665908813477,
      "logits/rejected": 8.970166206359863,
      "logps/chosen": -43.786033630371094,
      "logps/rejected": -45.51081466674805,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -43.786033630371094,
      "rewards/margins": 1.7247825860977173,
      "rewards/rejected": -45.51081466674805,
      "step": 5210
    },
    {
      "epoch": 2.7911021910018396,
      "grad_norm": 4.0256292204523195e-11,
      "learning_rate": 1.4609476715427226e-08,
      "logits/chosen": 9.315170288085938,
      "logits/rejected": 8.941254615783691,
      "logps/chosen": -42.23725509643555,
      "logps/rejected": -43.261512756347656,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -42.23725509643555,
      "rewards/margins": 1.0242549180984497,
      "rewards/rejected": -43.261512756347656,
      "step": 5215
    },
    {
      "epoch": 2.7937782237832414,
      "grad_norm": 2.557174067438597e-07,
      "learning_rate": 1.4238106709949792e-08,
      "logits/chosen": 9.222701072692871,
      "logits/rejected": 8.876856803894043,
      "logps/chosen": -43.900089263916016,
      "logps/rejected": -43.921295166015625,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -43.900089263916016,
      "rewards/margins": 0.021208714693784714,
      "rewards/rejected": -43.921295166015625,
      "step": 5220
    },
    {
      "epoch": 2.796454256564643,
      "grad_norm": 1.1313447034546127e-08,
      "learning_rate": 1.3871449568491511e-08,
      "logits/chosen": 8.978532791137695,
      "logits/rejected": 8.517302513122559,
      "logps/chosen": -44.594696044921875,
      "logps/rejected": -45.526573181152344,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.594696044921875,
      "rewards/margins": 0.9318796396255493,
      "rewards/rejected": -45.526573181152344,
      "step": 5225
    },
    {
      "epoch": 2.7991302893460444,
      "grad_norm": 1.160158799261448e-08,
      "learning_rate": 1.3509508848361606e-08,
      "logits/chosen": 9.403702735900879,
      "logits/rejected": 8.873385429382324,
      "logps/chosen": -43.27454376220703,
      "logps/rejected": -44.70935821533203,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -43.27454376220703,
      "rewards/margins": 1.434807538986206,
      "rewards/rejected": -44.70935821533203,
      "step": 5230
    },
    {
      "epoch": 2.801806322127446,
      "grad_norm": 7.820727053378319e-06,
      "learning_rate": 1.3152288061110517e-08,
      "logits/chosen": 9.201998710632324,
      "logits/rejected": 8.765900611877441,
      "logps/chosen": -42.991729736328125,
      "logps/rejected": -44.02509689331055,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.991729736328125,
      "rewards/margins": 1.033368706703186,
      "rewards/rejected": -44.02509689331055,
      "step": 5235
    },
    {
      "epoch": 2.804482354908848,
      "grad_norm": 1.7902472447983859e-10,
      "learning_rate": 1.2799790672495814e-08,
      "logits/chosen": 9.463716506958008,
      "logits/rejected": 8.737053871154785,
      "logps/chosen": -43.7899055480957,
      "logps/rejected": -45.37424850463867,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.7899055480957,
      "rewards/margins": 1.5843477249145508,
      "rewards/rejected": -45.37424850463867,
      "step": 5240
    },
    {
      "epoch": 2.807158387690249,
      "grad_norm": 3.8736349170294136e-12,
      "learning_rate": 1.2452020102448835e-08,
      "logits/chosen": 9.046751022338867,
      "logits/rejected": 8.856473922729492,
      "logps/chosen": -42.971893310546875,
      "logps/rejected": -43.707008361816406,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -42.971893310546875,
      "rewards/margins": 0.7351185083389282,
      "rewards/rejected": -43.707008361816406,
      "step": 5245
    },
    {
      "epoch": 2.8098344204716508,
      "grad_norm": 1.8362881789696736e-10,
      "learning_rate": 1.2108979725041103e-08,
      "logits/chosen": 9.25361442565918,
      "logits/rejected": 8.988142967224121,
      "logps/chosen": -42.874149322509766,
      "logps/rejected": -43.95782470703125,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.874149322509766,
      "rewards/margins": 1.0836725234985352,
      "rewards/rejected": -43.95782470703125,
      "step": 5250
    },
    {
      "epoch": 2.8125104532530525,
      "grad_norm": 6.89174656477691e-08,
      "learning_rate": 1.1770672868451958e-08,
      "logits/chosen": 9.159260749816895,
      "logits/rejected": 8.536584854125977,
      "logps/chosen": -42.166114807128906,
      "logps/rejected": -43.92035675048828,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.166114807128906,
      "rewards/margins": 1.7542474269866943,
      "rewards/rejected": -43.92035675048828,
      "step": 5255
    },
    {
      "epoch": 2.8151864860344538,
      "grad_norm": 9.116662477275866e-08,
      "learning_rate": 1.1437102814935872e-08,
      "logits/chosen": 8.747381210327148,
      "logits/rejected": 8.502212524414062,
      "logps/chosen": -43.68171691894531,
      "logps/rejected": -43.559906005859375,
      "loss": 0.0002,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.68171691894531,
      "rewards/margins": -0.1218104362487793,
      "rewards/rejected": -43.559906005859375,
      "step": 5260
    },
    {
      "epoch": 2.8178625188158555,
      "grad_norm": 6.907126013085905e-08,
      "learning_rate": 1.1108272800791018e-08,
      "logits/chosen": 9.698392868041992,
      "logits/rejected": 8.744329452514648,
      "logps/chosen": -42.207672119140625,
      "logps/rejected": -44.34783935546875,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.207672119140625,
      "rewards/margins": 2.140167474746704,
      "rewards/rejected": -44.34783935546875,
      "step": 5265
    },
    {
      "epoch": 2.820538551597257,
      "grad_norm": 6.983112331400282e-09,
      "learning_rate": 1.078418601632769e-08,
      "logits/chosen": 9.046021461486816,
      "logits/rejected": 8.639293670654297,
      "logps/chosen": -42.759029388427734,
      "logps/rejected": -43.713539123535156,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.759029388427734,
      "rewards/margins": 0.9545109868049622,
      "rewards/rejected": -43.713539123535156,
      "step": 5270
    },
    {
      "epoch": 2.8232145843786585,
      "grad_norm": 1.4433310476687422e-10,
      "learning_rate": 1.0464845605837159e-08,
      "logits/chosen": 9.016949653625488,
      "logits/rejected": 8.667830467224121,
      "logps/chosen": -43.385009765625,
      "logps/rejected": -44.63142776489258,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.385009765625,
      "rewards/margins": 1.246415376663208,
      "rewards/rejected": -44.63142776489258,
      "step": 5275
    },
    {
      "epoch": 2.82589061716006,
      "grad_norm": 0.00019095416438492458,
      "learning_rate": 1.0150254667561642e-08,
      "logits/chosen": 9.099616050720215,
      "logits/rejected": 8.377941131591797,
      "logps/chosen": -43.24871826171875,
      "logps/rejected": -44.87980651855469,
      "loss": 0.0023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.24871826171875,
      "rewards/margins": 1.6310901641845703,
      "rewards/rejected": -44.87980651855469,
      "step": 5280
    },
    {
      "epoch": 2.828566649941462,
      "grad_norm": 1.527120312796905e-09,
      "learning_rate": 9.840416253663719e-09,
      "logits/chosen": 9.420157432556152,
      "logits/rejected": 9.100214004516602,
      "logps/chosen": -43.914451599121094,
      "logps/rejected": -45.43206787109375,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.914451599121094,
      "rewards/margins": 1.5176138877868652,
      "rewards/rejected": -45.43206787109375,
      "step": 5285
    },
    {
      "epoch": 2.8312426827228636,
      "grad_norm": 3.5039508246119844e-13,
      "learning_rate": 9.535333370197074e-09,
      "logits/chosen": 9.187828063964844,
      "logits/rejected": 8.611418724060059,
      "logps/chosen": -43.67717361450195,
      "logps/rejected": -44.80834197998047,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.67717361450195,
      "rewards/margins": 1.1311668157577515,
      "rewards/rejected": -44.80834197998047,
      "step": 5290
    },
    {
      "epoch": 2.833918715504265,
      "grad_norm": 5.882714809599216e-07,
      "learning_rate": 9.23500897707713e-09,
      "logits/chosen": 9.386972427368164,
      "logits/rejected": 8.771082878112793,
      "logps/chosen": -43.3702507019043,
      "logps/rejected": -44.68819046020508,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.3702507019043,
      "rewards/margins": 1.3179371356964111,
      "rewards/rejected": -44.68819046020508,
      "step": 5295
    },
    {
      "epoch": 2.8365947482856666,
      "grad_norm": 6.470387932983341e-11,
      "learning_rate": 8.939445988052574e-09,
      "logits/chosen": 9.07305908203125,
      "logits/rejected": 8.82376766204834,
      "logps/chosen": -43.485137939453125,
      "logps/rejected": -43.979042053222656,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.485137939453125,
      "rewards/margins": 0.4939079284667969,
      "rewards/rejected": -43.979042053222656,
      "step": 5300
    },
    {
      "epoch": 2.839270781067068,
      "grad_norm": 1.6241058592339942e-09,
      "learning_rate": 8.648647270676656e-09,
      "logits/chosen": 8.844246864318848,
      "logits/rejected": 8.372082710266113,
      "logps/chosen": -42.29883575439453,
      "logps/rejected": -43.836509704589844,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.29883575439453,
      "rewards/margins": 1.537675380706787,
      "rewards/rejected": -43.836509704589844,
      "step": 5305
    },
    {
      "epoch": 2.8419468138484696,
      "grad_norm": 3.849104408977363e-14,
      "learning_rate": 8.362615646279991e-09,
      "logits/chosen": 9.875150680541992,
      "logits/rejected": 8.860733032226562,
      "logps/chosen": -44.44951248168945,
      "logps/rejected": -45.70476150512695,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.44951248168945,
      "rewards/margins": 1.255252480506897,
      "rewards/rejected": -45.70476150512695,
      "step": 5310
    },
    {
      "epoch": 2.8446228466298713,
      "grad_norm": 1.9994326140079131e-10,
      "learning_rate": 8.081353889942466e-09,
      "logits/chosen": 9.169601440429688,
      "logits/rejected": 8.663008689880371,
      "logps/chosen": -43.47755432128906,
      "logps/rejected": -45.06774139404297,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.47755432128906,
      "rewards/margins": 1.5901862382888794,
      "rewards/rejected": -45.06774139404297,
      "step": 5315
    },
    {
      "epoch": 2.847298879411273,
      "grad_norm": 1.2209391931202481e-09,
      "learning_rate": 7.804864730467042e-09,
      "logits/chosen": 9.036087989807129,
      "logits/rejected": 8.97172737121582,
      "logps/chosen": -43.215675354003906,
      "logps/rejected": -44.50541687011719,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.215675354003906,
      "rewards/margins": 1.2897390127182007,
      "rewards/rejected": -44.50541687011719,
      "step": 5320
    },
    {
      "epoch": 2.8499749121926743,
      "grad_norm": 1.1100234859355303e-09,
      "learning_rate": 7.533150850352665e-09,
      "logits/chosen": 9.126693725585938,
      "logits/rejected": 8.463194847106934,
      "logps/chosen": -44.90612030029297,
      "logps/rejected": -44.989322662353516,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -44.90612030029297,
      "rewards/margins": 0.08319838345050812,
      "rewards/rejected": -44.989322662353516,
      "step": 5325
    },
    {
      "epoch": 2.852650944974076,
      "grad_norm": 5.4843370305422565e-09,
      "learning_rate": 7.2662148857686175e-09,
      "logits/chosen": 8.544363021850586,
      "logits/rejected": 8.329607009887695,
      "logps/chosen": -42.94819259643555,
      "logps/rejected": -43.7733154296875,
      "loss": 0.0004,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.94819259643555,
      "rewards/margins": 0.8251211047172546,
      "rewards/rejected": -43.7733154296875,
      "step": 5330
    },
    {
      "epoch": 2.8553269777554773,
      "grad_norm": 2.143171486734352e-09,
      "learning_rate": 7.0040594265287635e-09,
      "logits/chosen": 8.722869873046875,
      "logits/rejected": 8.899015426635742,
      "logps/chosen": -43.56779098510742,
      "logps/rejected": -44.39909744262695,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -43.56779098510742,
      "rewards/margins": 0.8313077688217163,
      "rewards/rejected": -44.39909744262695,
      "step": 5335
    },
    {
      "epoch": 2.858003010536879,
      "grad_norm": 5.4126539382297734e-12,
      "learning_rate": 6.746687016066566e-09,
      "logits/chosen": 9.215690612792969,
      "logits/rejected": 9.012933731079102,
      "logps/chosen": -43.967124938964844,
      "logps/rejected": -44.7640495300293,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.967124938964844,
      "rewards/margins": 0.7969287037849426,
      "rewards/rejected": -44.7640495300293,
      "step": 5340
    },
    {
      "epoch": 2.8606790433182807,
      "grad_norm": 1.0171548077762303e-08,
      "learning_rate": 6.494100151410276e-09,
      "logits/chosen": 9.249914169311523,
      "logits/rejected": 8.525673866271973,
      "logps/chosen": -42.68964385986328,
      "logps/rejected": -44.34496307373047,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -42.68964385986328,
      "rewards/margins": 1.655322790145874,
      "rewards/rejected": -44.34496307373047,
      "step": 5345
    },
    {
      "epoch": 2.8633550760996824,
      "grad_norm": 2.4436930405833195e-08,
      "learning_rate": 6.246301283158728e-09,
      "logits/chosen": 8.896940231323242,
      "logits/rejected": 8.847696304321289,
      "logps/chosen": -43.179649353027344,
      "logps/rejected": -43.715843200683594,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.179649353027344,
      "rewards/margins": 0.5361932516098022,
      "rewards/rejected": -43.715843200683594,
      "step": 5350
    },
    {
      "epoch": 2.8660311088810837,
      "grad_norm": 1.3190865904739134e-09,
      "learning_rate": 6.0032928154576944e-09,
      "logits/chosen": 9.236913681030273,
      "logits/rejected": 9.100135803222656,
      "logps/chosen": -42.778934478759766,
      "logps/rejected": -44.05268096923828,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.778934478759766,
      "rewards/margins": 1.2737449407577515,
      "rewards/rejected": -44.05268096923828,
      "step": 5355
    },
    {
      "epoch": 2.8687071416624854,
      "grad_norm": 1.5797463887560218e-09,
      "learning_rate": 5.76507710597629e-09,
      "logits/chosen": 9.130548477172852,
      "logits/rejected": 8.44378662109375,
      "logps/chosen": -42.159114837646484,
      "logps/rejected": -44.34185028076172,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.159114837646484,
      "rewards/margins": 2.1827340126037598,
      "rewards/rejected": -44.34185028076172,
      "step": 5360
    },
    {
      "epoch": 2.8713831744438867,
      "grad_norm": 1.2612782631810013e-11,
      "learning_rate": 5.531656465884438e-09,
      "logits/chosen": 8.986358642578125,
      "logits/rejected": 8.461038589477539,
      "logps/chosen": -42.87377166748047,
      "logps/rejected": -44.58510971069336,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -42.87377166748047,
      "rewards/margins": 1.7113420963287354,
      "rewards/rejected": -44.58510971069336,
      "step": 5365
    },
    {
      "epoch": 2.8740592072252884,
      "grad_norm": 1.2347539369113793e-11,
      "learning_rate": 5.303033159830217e-09,
      "logits/chosen": 8.828839302062988,
      "logits/rejected": 8.831342697143555,
      "logps/chosen": -42.833251953125,
      "logps/rejected": -44.327335357666016,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.833251953125,
      "rewards/margins": 1.494086503982544,
      "rewards/rejected": -44.327335357666016,
      "step": 5370
    },
    {
      "epoch": 2.87673524000669,
      "grad_norm": 2.1706719966774322e-13,
      "learning_rate": 5.079209405917939e-09,
      "logits/chosen": 8.819482803344727,
      "logits/rejected": 8.773767471313477,
      "logps/chosen": -43.446250915527344,
      "logps/rejected": -45.201904296875,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -43.446250915527344,
      "rewards/margins": 1.7556512355804443,
      "rewards/rejected": -45.201904296875,
      "step": 5375
    },
    {
      "epoch": 2.879411272788092,
      "grad_norm": 5.11554878723892e-10,
      "learning_rate": 4.860187375686664e-09,
      "logits/chosen": 9.458517074584961,
      "logits/rejected": 8.459425926208496,
      "logps/chosen": -42.429019927978516,
      "logps/rejected": -44.0147590637207,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -42.429019927978516,
      "rewards/margins": 1.5857442617416382,
      "rewards/rejected": -44.0147590637207,
      "step": 5380
    },
    {
      "epoch": 2.882087305569493,
      "grad_norm": 7.844521320531377e-12,
      "learning_rate": 4.64596919408905e-09,
      "logits/chosen": 8.818659782409668,
      "logits/rejected": 8.798497200012207,
      "logps/chosen": -41.92902374267578,
      "logps/rejected": -43.45621871948242,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.92902374267578,
      "rewards/margins": 1.5271942615509033,
      "rewards/rejected": -43.45621871948242,
      "step": 5385
    },
    {
      "epoch": 2.884763338350895,
      "grad_norm": 4.961375735543267e-06,
      "learning_rate": 4.436556939470814e-09,
      "logits/chosen": 9.121790885925293,
      "logits/rejected": 8.664724349975586,
      "logps/chosen": -44.26715087890625,
      "logps/rejected": -44.96699905395508,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.26715087890625,
      "rewards/margins": 0.6998492479324341,
      "rewards/rejected": -44.96699905395508,
      "step": 5390
    },
    {
      "epoch": 2.887439371132296,
      "grad_norm": 6.458950029941331e-07,
      "learning_rate": 4.23195264355064e-09,
      "logits/chosen": 9.128458023071289,
      "logits/rejected": 8.465438842773438,
      "logps/chosen": -42.92496109008789,
      "logps/rejected": -44.21221923828125,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.92496109008789,
      "rewards/margins": 1.2872591018676758,
      "rewards/rejected": -44.21221923828125,
      "step": 5395
    },
    {
      "epoch": 2.890115403913698,
      "grad_norm": 9.296836917399921e-07,
      "learning_rate": 4.032158291400245e-09,
      "logits/chosen": 9.210552215576172,
      "logits/rejected": 8.299796104431152,
      "logps/chosen": -42.495174407958984,
      "logps/rejected": -44.12283706665039,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.495174407958984,
      "rewards/margins": 1.6276592016220093,
      "rewards/rejected": -44.12283706665039,
      "step": 5400
    },
    {
      "epoch": 2.8927914366950995,
      "grad_norm": 0.00015403173900330994,
      "learning_rate": 3.837175821425398e-09,
      "logits/chosen": 8.843391418457031,
      "logits/rejected": 8.573896408081055,
      "logps/chosen": -43.014854431152344,
      "logps/rejected": -43.65604019165039,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.014854431152344,
      "rewards/margins": 0.6411841511726379,
      "rewards/rejected": -43.65604019165039,
      "step": 5405
    },
    {
      "epoch": 2.8954674694765012,
      "grad_norm": 8.547261274400296e-07,
      "learning_rate": 3.6470071253467683e-09,
      "logits/chosen": 9.146087646484375,
      "logits/rejected": 8.660335540771484,
      "logps/chosen": -42.8497314453125,
      "logps/rejected": -44.10491180419922,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.8497314453125,
      "rewards/margins": 1.255183219909668,
      "rewards/rejected": -44.10491180419922,
      "step": 5410
    },
    {
      "epoch": 2.8981435022579025,
      "grad_norm": 1.3361023416888584e-10,
      "learning_rate": 3.461654048181939e-09,
      "logits/chosen": 9.334836959838867,
      "logits/rejected": 8.748196601867676,
      "logps/chosen": -43.159141540527344,
      "logps/rejected": -44.73859786987305,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.159141540527344,
      "rewards/margins": 1.5794521570205688,
      "rewards/rejected": -44.73859786987305,
      "step": 5415
    },
    {
      "epoch": 2.9008195350393042,
      "grad_norm": 1.6339961467233275e-11,
      "learning_rate": 3.281118388227255e-09,
      "logits/chosen": 9.09322452545166,
      "logits/rejected": 8.83458423614502,
      "logps/chosen": -43.47234344482422,
      "logps/rejected": -43.72627639770508,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.47234344482422,
      "rewards/margins": 0.25393232703208923,
      "rewards/rejected": -43.72627639770508,
      "step": 5420
    },
    {
      "epoch": 2.903495567820706,
      "grad_norm": 9.958094605143179e-11,
      "learning_rate": 3.1054018970405048e-09,
      "logits/chosen": 9.123544692993164,
      "logits/rejected": 8.559076309204102,
      "logps/chosen": -43.154457092285156,
      "logps/rejected": -44.35981750488281,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.154457092285156,
      "rewards/margins": 1.2053630352020264,
      "rewards/rejected": -44.35981750488281,
      "step": 5425
    },
    {
      "epoch": 2.906171600602107,
      "grad_norm": 2.4276863648133436e-06,
      "learning_rate": 2.9345062794238207e-09,
      "logits/chosen": 9.339111328125,
      "logits/rejected": 8.790606498718262,
      "logps/chosen": -43.166709899902344,
      "logps/rejected": -44.22993850708008,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.166709899902344,
      "rewards/margins": 1.0632320642471313,
      "rewards/rejected": -44.22993850708008,
      "step": 5430
    },
    {
      "epoch": 2.908847633383509,
      "grad_norm": 1.2759251064601855e-13,
      "learning_rate": 2.7684331934072492e-09,
      "logits/chosen": 9.134610176086426,
      "logits/rejected": 8.85561752319336,
      "logps/chosen": -42.64212417602539,
      "logps/rejected": -43.90853500366211,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.64212417602539,
      "rewards/margins": 1.2664151191711426,
      "rewards/rejected": -43.90853500366211,
      "step": 5435
    },
    {
      "epoch": 2.9115236661649107,
      "grad_norm": 3.105596137804807e-08,
      "learning_rate": 2.6071842502326526e-09,
      "logits/chosen": 9.124176025390625,
      "logits/rejected": 8.62415885925293,
      "logps/chosen": -42.54281234741211,
      "logps/rejected": -43.394012451171875,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.54281234741211,
      "rewards/margins": 0.8511954545974731,
      "rewards/rejected": -43.394012451171875,
      "step": 5440
    },
    {
      "epoch": 2.9141996989463124,
      "grad_norm": 6.404810854747346e-06,
      "learning_rate": 2.450761014337888e-09,
      "logits/chosen": 8.63016414642334,
      "logits/rejected": 8.456235885620117,
      "logps/chosen": -42.82942199707031,
      "logps/rejected": -43.71613311767578,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.82942199707031,
      "rewards/margins": 0.886710524559021,
      "rewards/rejected": -43.71613311767578,
      "step": 5445
    },
    {
      "epoch": 2.9168757317277136,
      "grad_norm": 5.547656492059674e-07,
      "learning_rate": 2.299165003341985e-09,
      "logits/chosen": 8.547208786010742,
      "logits/rejected": 8.238048553466797,
      "logps/chosen": -42.25947189331055,
      "logps/rejected": -42.96728515625,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -42.25947189331055,
      "rewards/margins": 0.7078157067298889,
      "rewards/rejected": -42.96728515625,
      "step": 5450
    },
    {
      "epoch": 2.9195517645091154,
      "grad_norm": 6.663820388596482e-06,
      "learning_rate": 2.1523976880299945e-09,
      "logits/chosen": 9.08293342590332,
      "logits/rejected": 8.70988655090332,
      "logps/chosen": -42.87151336669922,
      "logps/rejected": -44.23230743408203,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.87151336669922,
      "rewards/margins": 1.3607940673828125,
      "rewards/rejected": -44.23230743408203,
      "step": 5455
    },
    {
      "epoch": 2.9222277972905166,
      "grad_norm": 2.6349918700593994e-12,
      "learning_rate": 2.010460492339161e-09,
      "logits/chosen": 8.973346710205078,
      "logits/rejected": 8.453689575195312,
      "logps/chosen": -42.803321838378906,
      "logps/rejected": -44.63463592529297,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -42.803321838378906,
      "rewards/margins": 1.8313194513320923,
      "rewards/rejected": -44.63463592529297,
      "step": 5460
    },
    {
      "epoch": 2.9249038300719183,
      "grad_norm": 4.86117986466598e-05,
      "learning_rate": 1.8733547933446614e-09,
      "logits/chosen": 9.281246185302734,
      "logits/rejected": 8.57530689239502,
      "logps/chosen": -42.82166290283203,
      "logps/rejected": -43.41827392578125,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.82166290283203,
      "rewards/margins": 0.596615731716156,
      "rewards/rejected": -43.41827392578125,
      "step": 5465
    },
    {
      "epoch": 2.92757986285332,
      "grad_norm": 2.4918971089722022e-09,
      "learning_rate": 1.7410819212467231e-09,
      "logits/chosen": 9.121832847595215,
      "logits/rejected": 8.893570899963379,
      "logps/chosen": -44.752296447753906,
      "logps/rejected": -45.21686553955078,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.752296447753906,
      "rewards/margins": 0.4645637571811676,
      "rewards/rejected": -45.21686553955078,
      "step": 5470
    },
    {
      "epoch": 2.9302558956347218,
      "grad_norm": 1.1491199409456674e-11,
      "learning_rate": 1.613643159357192e-09,
      "logits/chosen": 9.142816543579102,
      "logits/rejected": 9.115776062011719,
      "logps/chosen": -43.880592346191406,
      "logps/rejected": -44.0962028503418,
      "loss": 0.0,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -43.880592346191406,
      "rewards/margins": 0.2156103551387787,
      "rewards/rejected": -44.0962028503418,
      "step": 5475
    },
    {
      "epoch": 2.932931928416123,
      "grad_norm": 1.1122140933028381e-08,
      "learning_rate": 1.4910397440875967e-09,
      "logits/chosen": 9.083062171936035,
      "logits/rejected": 8.739364624023438,
      "logps/chosen": -43.33385467529297,
      "logps/rejected": -44.41044616699219,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.33385467529297,
      "rewards/margins": 1.0765869617462158,
      "rewards/rejected": -44.41044616699219,
      "step": 5480
    },
    {
      "epoch": 2.9356079611975248,
      "grad_norm": 7.680496960003502e-10,
      "learning_rate": 1.3732728649368253e-09,
      "logits/chosen": 8.916650772094727,
      "logits/rejected": 8.185681343078613,
      "logps/chosen": -43.41474914550781,
      "logps/rejected": -44.25482940673828,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -43.41474914550781,
      "rewards/margins": 0.8400863409042358,
      "rewards/rejected": -44.25482940673828,
      "step": 5485
    },
    {
      "epoch": 2.938283993978926,
      "grad_norm": 1.41509761269715e-10,
      "learning_rate": 1.260343664479524e-09,
      "logits/chosen": 8.641057968139648,
      "logits/rejected": 8.601696968078613,
      "logps/chosen": -43.54811477661133,
      "logps/rejected": -43.978782653808594,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -43.54811477661133,
      "rewards/margins": 0.43066921830177307,
      "rewards/rejected": -43.978782653808594,
      "step": 5490
    },
    {
      "epoch": 2.9409600267603278,
      "grad_norm": 6.619161244267428e-06,
      "learning_rate": 1.1522532383554384e-09,
      "logits/chosen": 9.290199279785156,
      "logits/rejected": 8.573363304138184,
      "logps/chosen": -43.536781311035156,
      "logps/rejected": -44.43586349487305,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.536781311035156,
      "rewards/margins": 0.8990872502326965,
      "rewards/rejected": -44.43586349487305,
      "step": 5495
    },
    {
      "epoch": 2.9436360595417295,
      "grad_norm": 4.2023568582106304e-10,
      "learning_rate": 1.049002635258256e-09,
      "logits/chosen": 8.85502815246582,
      "logits/rejected": 8.50245475769043,
      "logps/chosen": -43.516456604003906,
      "logps/rejected": -44.108821868896484,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.516456604003906,
      "rewards/margins": 0.5923608541488647,
      "rewards/rejected": -44.108821868896484,
      "step": 5500
    },
    {
      "epoch": 2.946312092323131,
      "grad_norm": 7.862844538251259e-05,
      "learning_rate": 9.505928569258358e-10,
      "logits/chosen": 8.895758628845215,
      "logits/rejected": 8.956740379333496,
      "logps/chosen": -43.34341812133789,
      "logps/rejected": -44.25432586669922,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -43.34341812133789,
      "rewards/margins": 0.9109099507331848,
      "rewards/rejected": -44.25432586669922,
      "step": 5505
    },
    {
      "epoch": 2.9489881251045325,
      "grad_norm": 3.3511889003294474e-11,
      "learning_rate": 8.57024858130273e-10,
      "logits/chosen": 9.06314754486084,
      "logits/rejected": 8.55306625366211,
      "logps/chosen": -43.68933868408203,
      "logps/rejected": -44.623023986816406,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.68933868408203,
      "rewards/margins": 0.9336822628974915,
      "rewards/rejected": -44.623023986816406,
      "step": 5510
    },
    {
      "epoch": 2.951664157885934,
      "grad_norm": 3.091503354543624e-12,
      "learning_rate": 7.682995466686826e-10,
      "logits/chosen": 9.43451976776123,
      "logits/rejected": 8.934001922607422,
      "logps/chosen": -43.223777770996094,
      "logps/rejected": -44.37071990966797,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.223777770996094,
      "rewards/margins": 1.1469380855560303,
      "rewards/rejected": -44.37071990966797,
      "step": 5515
    },
    {
      "epoch": 2.9543401906673354,
      "grad_norm": 1.620901653731338e-10,
      "learning_rate": 6.844177833543741e-10,
      "logits/chosen": 9.181394577026367,
      "logits/rejected": 8.99818229675293,
      "logps/chosen": -43.271820068359375,
      "logps/rejected": -44.982200622558594,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.271820068359375,
      "rewards/margins": 1.7103798389434814,
      "rewards/rejected": -44.982200622558594,
      "step": 5520
    },
    {
      "epoch": 2.957016223448737,
      "grad_norm": 7.344292945024075e-13,
      "learning_rate": 6.053803820087467e-10,
      "logits/chosen": 9.069437026977539,
      "logits/rejected": 8.609970092773438,
      "logps/chosen": -44.08431625366211,
      "logps/rejected": -45.02243423461914,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -44.08431625366211,
      "rewards/margins": 0.9381159543991089,
      "rewards/rejected": -45.02243423461914,
      "step": 5525
    },
    {
      "epoch": 2.959692256230139,
      "grad_norm": 2.768078971846132e-08,
      "learning_rate": 5.311881094528514e-10,
      "logits/chosen": 9.123296737670898,
      "logits/rejected": 8.427828788757324,
      "logps/chosen": -43.0762825012207,
      "logps/rejected": -44.399314880371094,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.0762825012207,
      "rewards/margins": 1.3230338096618652,
      "rewards/rejected": -44.399314880371094,
      "step": 5530
    },
    {
      "epoch": 2.9623682890115406,
      "grad_norm": 3.89638696647581e-05,
      "learning_rate": 4.6184168550050806e-10,
      "logits/chosen": 8.684124946594238,
      "logits/rejected": 8.624351501464844,
      "logps/chosen": -42.91204071044922,
      "logps/rejected": -43.33562088012695,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.91204071044922,
      "rewards/margins": 0.4235750138759613,
      "rewards/rejected": -43.33562088012695,
      "step": 5535
    },
    {
      "epoch": 2.965044321792942,
      "grad_norm": 1.7424728723119498e-09,
      "learning_rate": 3.973417829510328e-10,
      "logits/chosen": 9.220220565795898,
      "logits/rejected": 8.75816822052002,
      "logps/chosen": -42.318565368652344,
      "logps/rejected": -43.9902229309082,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.318565368652344,
      "rewards/margins": 1.671654462814331,
      "rewards/rejected": -43.9902229309082,
      "step": 5540
    },
    {
      "epoch": 2.9677203545743436,
      "grad_norm": 4.9638784506663666e-11,
      "learning_rate": 3.3768902758274377e-10,
      "logits/chosen": 8.967167854309082,
      "logits/rejected": 8.66887378692627,
      "logps/chosen": -41.842193603515625,
      "logps/rejected": -43.09015655517578,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.842193603515625,
      "rewards/margins": 1.2479666471481323,
      "rewards/rejected": -43.09015655517578,
      "step": 5545
    },
    {
      "epoch": 2.970396387355745,
      "grad_norm": 1.0907062190140083e-07,
      "learning_rate": 2.8288399814691e-10,
      "logits/chosen": 8.862554550170898,
      "logits/rejected": 8.886552810668945,
      "logps/chosen": -43.24170684814453,
      "logps/rejected": -44.441734313964844,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.24170684814453,
      "rewards/margins": 1.200028657913208,
      "rewards/rejected": -44.441734313964844,
      "step": 5550
    },
    {
      "epoch": 2.9730724201371466,
      "grad_norm": 2.474885916988663e-09,
      "learning_rate": 2.3292722636220066e-10,
      "logits/chosen": 9.124627113342285,
      "logits/rejected": 8.516072273254395,
      "logps/chosen": -42.52320098876953,
      "logps/rejected": -44.84089279174805,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -42.52320098876953,
      "rewards/margins": 2.317686080932617,
      "rewards/rejected": -44.84089279174805,
      "step": 5555
    },
    {
      "epoch": 2.9757484529185483,
      "grad_norm": 6.353003881160805e-11,
      "learning_rate": 1.8781919690946668e-10,
      "logits/chosen": 8.577714920043945,
      "logits/rejected": 8.480096817016602,
      "logps/chosen": -44.2556266784668,
      "logps/rejected": -44.30500411987305,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -44.2556266784668,
      "rewards/margins": 0.04937465116381645,
      "rewards/rejected": -44.30500411987305,
      "step": 5560
    },
    {
      "epoch": 2.97842448569995,
      "grad_norm": 8.746418768118171e-08,
      "learning_rate": 1.4756034742696711e-10,
      "logits/chosen": 9.16465950012207,
      "logits/rejected": 8.840929985046387,
      "logps/chosen": -42.676368713378906,
      "logps/rejected": -43.760292053222656,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.676368713378906,
      "rewards/margins": 1.0839216709136963,
      "rewards/rejected": -43.760292053222656,
      "step": 5565
    },
    {
      "epoch": 2.9811005184813513,
      "grad_norm": 1.1753167945445568e-08,
      "learning_rate": 1.12151068506261e-10,
      "logits/chosen": 9.012102127075195,
      "logits/rejected": 8.687840461730957,
      "logps/chosen": -43.252235412597656,
      "logps/rejected": -44.85663986206055,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.252235412597656,
      "rewards/margins": 1.6044065952301025,
      "rewards/rejected": -44.85663986206055,
      "step": 5570
    },
    {
      "epoch": 2.983776551262753,
      "grad_norm": 1.5292187555844035e-09,
      "learning_rate": 8.159170368826629e-11,
      "logits/chosen": 8.963420867919922,
      "logits/rejected": 8.400527000427246,
      "logps/chosen": -42.38309097290039,
      "logps/rejected": -44.467132568359375,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.38309097290039,
      "rewards/margins": 2.0840423107147217,
      "rewards/rejected": -44.467132568359375,
      "step": 5575
    },
    {
      "epoch": 2.9864525840441547,
      "grad_norm": 1.941827391079566e-09,
      "learning_rate": 5.588254946015114e-11,
      "logits/chosen": 9.213991165161133,
      "logits/rejected": 8.347885131835938,
      "logps/chosen": -42.460777282714844,
      "logps/rejected": -44.171546936035156,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.460777282714844,
      "rewards/margins": 1.710771918296814,
      "rewards/rejected": -44.171546936035156,
      "step": 5580
    },
    {
      "epoch": 2.989128616825556,
      "grad_norm": 8.721330057929216e-12,
      "learning_rate": 3.502385525216978e-11,
      "logits/chosen": 8.794221878051758,
      "logits/rejected": 8.234247207641602,
      "logps/chosen": -43.3101921081543,
      "logps/rejected": -44.4868049621582,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.3101921081543,
      "rewards/margins": 1.176613450050354,
      "rewards/rejected": -44.4868049621582,
      "step": 5585
    },
    {
      "epoch": 2.9918046496069577,
      "grad_norm": 3.3914969676161003e-09,
      "learning_rate": 1.901582343555308e-11,
      "logits/chosen": 8.635357856750488,
      "logits/rejected": 8.538853645324707,
      "logps/chosen": -42.34468078613281,
      "logps/rejected": -43.14370346069336,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.34468078613281,
      "rewards/margins": 0.7990256547927856,
      "rewards/rejected": -43.14370346069336,
      "step": 5590
    },
    {
      "epoch": 2.9944806823883594,
      "grad_norm": 1.1978369083992432e-08,
      "learning_rate": 7.858609320232634e-12,
      "logits/chosen": 8.80547046661377,
      "logits/rejected": 8.196683883666992,
      "logps/chosen": -43.26044845581055,
      "logps/rejected": -45.38365936279297,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -43.26044845581055,
      "rewards/margins": 2.123211622238159,
      "rewards/rejected": -45.38365936279297,
      "step": 5595
    },
    {
      "epoch": 2.9971567151697607,
      "grad_norm": 6.715830344165064e-13,
      "learning_rate": 1.5523211535639624e-12,
      "logits/chosen": 9.053936958312988,
      "logits/rejected": 8.509064674377441,
      "logps/chosen": -43.21258544921875,
      "logps/rejected": -44.620113372802734,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.21258544921875,
      "rewards/margins": 1.4075287580490112,
      "rewards/rejected": -44.620113372802734,
      "step": 5600
    },
    {
      "epoch": 2.9971567151697607,
      "eval_logits/chosen": 7.539320468902588,
      "eval_logits/rejected": 7.256679534912109,
      "eval_logps/chosen": -42.77936935424805,
      "eval_logps/rejected": -43.914947509765625,
      "eval_loss": 1.999994623247403e-07,
      "eval_rewards/accuracies": 0.5667656064033508,
      "eval_rewards/chosen": -42.77936935424805,
      "eval_rewards/margins": 1.1355794668197632,
      "eval_rewards/rejected": -43.914947509765625,
      "eval_runtime": 41.286,
      "eval_samples_per_second": 32.578,
      "eval_steps_per_second": 8.163,
      "step": 5600
    },
    {
      "epoch": 2.999297541394882,
      "step": 5604,
      "total_flos": 0.0,
      "train_loss": 0.010826070491851839,
      "train_runtime": 30412.7115,
      "train_samples_per_second": 5.898,
      "train_steps_per_second": 0.184
    }
  ],
  "logging_steps": 5,
  "max_steps": 5604,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}