{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.1607717041800643,
  "eval_steps": 100,
  "global_step": 200,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 2.5493941690669297,
      "learning_rate": 4e-09,
      "logits/policy_chosen": -2.279445171356201,
      "logits/policy_rejected": -2.070641279220581,
      "logits/reference_chosen": -2.262451410293579,
      "logits/reference_rejected": -1.700181484222412,
      "logps/policy_chosen": -259.83294677734375,
      "logps/policy_rejected": -282.3813171386719,
      "logps/reference_chosen": -280.28363037109375,
      "logps/reference_rejected": -290.83868408203125,
      "loss": 0.6746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20450688898563385,
      "rewards/margins": 0.11993316560983658,
      "rewards/rejected": 0.08457372337579727,
      "step": 1,
      "tokens/chosen_response_len_mean": 528.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 444.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2490732615654547,
      "learning_rate": 8e-09,
      "logits/policy_chosen": -2.745072364807129,
      "logits/policy_rejected": -2.607792854309082,
      "logits/reference_chosen": -2.701054096221924,
      "logits/reference_rejected": -2.529007911682129,
      "logps/policy_chosen": -147.3030242919922,
      "logps/policy_rejected": -197.99253845214844,
      "logps/reference_chosen": -156.72518920898438,
      "logps/reference_rejected": -206.69471740722656,
      "loss": 0.6863,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09422144293785095,
      "rewards/margins": 0.0071996524930000305,
      "rewards/rejected": 0.08702179044485092,
      "step": 2,
      "tokens/chosen_response_len_mean": 298.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 277.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2889093862107543,
      "learning_rate": 1.2e-08,
      "logits/policy_chosen": -2.882375478744507,
      "logits/policy_rejected": -2.992877960205078,
      "logits/reference_chosen": -2.748257637023926,
      "logits/reference_rejected": -2.9701337814331055,
      "logps/policy_chosen": -105.87107849121094,
      "logps/policy_rejected": -148.20022583007812,
      "logps/reference_chosen": -112.85089111328125,
      "logps/reference_rejected": -155.7706298828125,
      "loss": 0.6832,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.06979808956384659,
      "rewards/margins": -0.005905959755182266,
      "rewards/rejected": 0.07570404559373856,
      "step": 3,
      "tokens/chosen_response_len_mean": 239.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 254.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2808153966501084,
      "learning_rate": 1.6e-08,
      "logits/policy_chosen": -2.9396884441375732,
      "logits/policy_rejected": -2.9384472370147705,
      "logits/reference_chosen": -2.9226865768432617,
      "logits/reference_rejected": -2.9309639930725098,
      "logps/policy_chosen": -123.63784790039062,
      "logps/policy_rejected": -180.25296020507812,
      "logps/reference_chosen": -129.93817138671875,
      "logps/reference_rejected": -184.21334838867188,
      "loss": 0.674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06300327926874161,
      "rewards/margins": 0.023399492725729942,
      "rewards/rejected": 0.039603784680366516,
      "step": 4,
      "tokens/chosen_response_len_mean": 298.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 297.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2660235876486974,
      "learning_rate": 2e-08,
      "logits/policy_chosen": -3.221891164779663,
      "logits/policy_rejected": -2.8190152645111084,
      "logits/reference_chosen": -3.207385778427124,
      "logits/reference_rejected": -2.6983580589294434,
      "logps/policy_chosen": -179.60354614257812,
      "logps/policy_rejected": -206.77960205078125,
      "logps/reference_chosen": -189.15216064453125,
      "logps/reference_rejected": -209.81790161132812,
      "loss": 0.6778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09548593312501907,
      "rewards/margins": 0.06510297954082489,
      "rewards/rejected": 0.03038296103477478,
      "step": 5,
      "tokens/chosen_response_len_mean": 440.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 232.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.0558578581082307,
      "learning_rate": 2.4e-08,
      "logits/policy_chosen": -2.4811384677886963,
      "logits/policy_rejected": -2.4067883491516113,
      "logits/reference_chosen": -2.330796003341675,
      "logits/reference_rejected": -2.4005446434020996,
      "logps/policy_chosen": -148.8319091796875,
      "logps/policy_rejected": -179.03482055664062,
      "logps/reference_chosen": -160.83175659179688,
      "logps/reference_rejected": -189.98275756835938,
      "loss": 0.6989,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11999829858541489,
      "rewards/margins": 0.010519090108573437,
      "rewards/rejected": 0.10947921127080917,
      "step": 6,
      "tokens/chosen_response_len_mean": 289.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 309.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.5447256930451734,
      "learning_rate": 2.8e-08,
      "logits/policy_chosen": -2.2599525451660156,
      "logits/policy_rejected": -2.190964698791504,
      "logits/reference_chosen": -2.1052117347717285,
      "logits/reference_rejected": -2.134941577911377,
      "logps/policy_chosen": -19.338191986083984,
      "logps/policy_rejected": -38.73615646362305,
      "logps/reference_chosen": -20.665983200073242,
      "logps/reference_rejected": -40.615394592285156,
      "loss": 0.684,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01327789481729269,
      "rewards/margins": -0.005514495074748993,
      "rewards/rejected": 0.018792390823364258,
      "step": 7,
      "tokens/chosen_response_len_mean": 34.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 47.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.967658233666017,
      "learning_rate": 3.2e-08,
      "logits/policy_chosen": -2.4715514183044434,
      "logits/policy_rejected": -2.4913477897644043,
      "logits/reference_chosen": -2.421693801879883,
      "logits/reference_rejected": -2.4608089923858643,
      "logps/policy_chosen": -29.782058715820312,
      "logps/policy_rejected": -64.6925048828125,
      "logps/reference_chosen": -32.50696563720703,
      "logps/reference_rejected": -66.7137222290039,
      "loss": 0.6712,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.027249068021774292,
      "rewards/margins": 0.00703694112598896,
      "rewards/rejected": 0.020212125033140182,
      "step": 8,
      "tokens/chosen_response_len_mean": 40.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 48.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 3.4470069507233316,
      "learning_rate": 3.6e-08,
      "logits/policy_chosen": -2.7727508544921875,
      "logits/policy_rejected": -2.718796730041504,
      "logits/reference_chosen": -2.676337957382202,
      "logits/reference_rejected": -2.616457223892212,
      "logps/policy_chosen": -108.26776123046875,
      "logps/policy_rejected": -218.6373748779297,
      "logps/reference_chosen": -118.67279815673828,
      "logps/reference_rejected": -223.27325439453125,
      "loss": 0.6796,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1040503978729248,
      "rewards/margins": 0.05769149586558342,
      "rewards/rejected": 0.046358898282051086,
      "step": 9,
      "tokens/chosen_response_len_mean": 252.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 222.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.304016636338041,
      "learning_rate": 4e-08,
      "logits/policy_chosen": -2.5566985607147217,
      "logits/policy_rejected": -2.5395240783691406,
      "logits/reference_chosen": -2.4825587272644043,
      "logits/reference_rejected": -2.498563051223755,
      "logps/policy_chosen": -168.69143676757812,
      "logps/policy_rejected": -287.2906494140625,
      "logps/reference_chosen": -181.2764129638672,
      "logps/reference_rejected": -299.7208557128906,
      "loss": 0.6885,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.1258496791124344,
      "rewards/margins": 0.001547420397400856,
      "rewards/rejected": 0.12430226802825928,
      "step": 10,
      "tokens/chosen_response_len_mean": 518.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 503.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.084799466815333,
      "learning_rate": 4.4e-08,
      "logits/policy_chosen": -2.7605199813842773,
      "logits/policy_rejected": -2.7186694145202637,
      "logits/reference_chosen": -2.7691569328308105,
      "logits/reference_rejected": -2.7842984199523926,
      "logps/policy_chosen": -145.88803100585938,
      "logps/policy_rejected": -219.43760681152344,
      "logps/reference_chosen": -155.90982055664062,
      "logps/reference_rejected": -227.8662109375,
      "loss": 0.6893,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1002180427312851,
      "rewards/margins": 0.015932004898786545,
      "rewards/rejected": 0.08428603410720825,
      "step": 11,
      "tokens/chosen_response_len_mean": 604.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 448.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.7628214791467736,
      "learning_rate": 4.8e-08,
      "logits/policy_chosen": -2.9070513248443604,
      "logits/policy_rejected": -2.7778186798095703,
      "logits/reference_chosen": -2.8774163722991943,
      "logits/reference_rejected": -2.6446313858032227,
      "logps/policy_chosen": -101.27095794677734,
      "logps/policy_rejected": -136.12225341796875,
      "logps/reference_chosen": -108.19136047363281,
      "logps/reference_rejected": -140.3483123779297,
      "loss": 0.6673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06920397281646729,
      "rewards/margins": 0.026943188160657883,
      "rewards/rejected": 0.0422607846558094,
      "step": 12,
      "tokens/chosen_response_len_mean": 339.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.1795487992104348,
      "learning_rate": 5.1999999999999996e-08,
      "logits/policy_chosen": -2.4187445640563965,
      "logits/policy_rejected": -2.845353364944458,
      "logits/reference_chosen": -2.3343820571899414,
      "logits/reference_rejected": -2.5970327854156494,
      "logps/policy_chosen": -130.312255859375,
      "logps/policy_rejected": -73.26902770996094,
      "logps/reference_chosen": -134.85186767578125,
      "logps/reference_rejected": -73.45825958251953,
      "loss": 0.6772,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.045396123081445694,
      "rewards/margins": 0.04350382089614868,
      "rewards/rejected": 0.0018922993913292885,
      "step": 13,
      "tokens/chosen_response_len_mean": 177.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 101.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.7157502561420026,
      "learning_rate": 5.6e-08,
      "logits/policy_chosen": -3.211674928665161,
      "logits/policy_rejected": -3.0040595531463623,
      "logits/reference_chosen": -3.2025675773620605,
      "logits/reference_rejected": -3.0620803833007812,
      "logps/policy_chosen": -195.99533081054688,
      "logps/policy_rejected": -242.1226806640625,
      "logps/reference_chosen": -208.40536499023438,
      "logps/reference_rejected": -252.9475860595703,
      "loss": 0.6781,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12410032004117966,
      "rewards/margins": 0.015851235017180443,
      "rewards/rejected": 0.10824908316135406,
      "step": 14,
      "tokens/chosen_response_len_mean": 543.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 408.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 3.50208846860952,
      "learning_rate": 6e-08,
      "logits/policy_chosen": -2.6727657318115234,
      "logits/policy_rejected": -2.5919432640075684,
      "logits/reference_chosen": -2.6382195949554443,
      "logits/reference_rejected": -2.417813777923584,
      "logps/policy_chosen": -170.3718719482422,
      "logps/policy_rejected": -223.43052673339844,
      "logps/reference_chosen": -191.32583618164062,
      "logps/reference_rejected": -242.33273315429688,
      "loss": 0.6686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.209539532661438,
      "rewards/margins": 0.020517464727163315,
      "rewards/rejected": 0.18902206420898438,
      "step": 15,
      "tokens/chosen_response_len_mean": 431.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 403.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 1.6207728056732267,
      "learning_rate": 6.4e-08,
      "logits/policy_chosen": -2.954475164413452,
      "logits/policy_rejected": -2.8040027618408203,
      "logits/reference_chosen": -2.9069364070892334,
      "logits/reference_rejected": -2.8010973930358887,
      "logps/policy_chosen": -54.05670166015625,
      "logps/policy_rejected": -109.65800476074219,
      "logps/reference_chosen": -55.35580825805664,
      "logps/reference_rejected": -108.90874481201172,
      "loss": 0.6824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.012991041876375675,
      "rewards/margins": 0.02048366516828537,
      "rewards/rejected": -0.007492623291909695,
      "step": 16,
      "tokens/chosen_response_len_mean": 89.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 96.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.409299627131989,
      "learning_rate": 6.8e-08,
      "logits/policy_chosen": -2.909968137741089,
      "logits/policy_rejected": -2.84816837310791,
      "logits/reference_chosen": -2.8810513019561768,
      "logits/reference_rejected": -2.6653146743774414,
      "logps/policy_chosen": -119.80767822265625,
      "logps/policy_rejected": -267.8531799316406,
      "logps/reference_chosen": -126.68125915527344,
      "logps/reference_rejected": -275.761474609375,
      "loss": 0.6995,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06873579323291779,
      "rewards/margins": -0.010347230359911919,
      "rewards/rejected": 0.07908301800489426,
      "step": 17,
      "tokens/chosen_response_len_mean": 314.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 251.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.703021449933624,
      "learning_rate": 7.2e-08,
      "logits/policy_chosen": -2.8886208534240723,
      "logits/policy_rejected": -2.603667974472046,
      "logits/reference_chosen": -2.8919739723205566,
      "logits/reference_rejected": -2.336148262023926,
      "logps/policy_chosen": -144.8170166015625,
      "logps/policy_rejected": -144.6788330078125,
      "logps/reference_chosen": -164.3677215576172,
      "logps/reference_rejected": -156.78842163085938,
      "loss": 0.679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1955069750547409,
      "rewards/margins": 0.07441116869449615,
      "rewards/rejected": 0.12109580636024475,
      "step": 18,
      "tokens/chosen_response_len_mean": 421.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 314.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 1.9691382131351773,
      "learning_rate": 7.599999999999999e-08,
      "logits/policy_chosen": -2.766261339187622,
      "logits/policy_rejected": -2.697242259979248,
      "logits/reference_chosen": -2.710810899734497,
      "logits/reference_rejected": -2.557218551635742,
      "logps/policy_chosen": -150.92379760742188,
      "logps/policy_rejected": -174.591064453125,
      "logps/reference_chosen": -160.51756286621094,
      "logps/reference_rejected": -185.50238037109375,
      "loss": 0.6851,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0959378108382225,
      "rewards/margins": -0.013175351545214653,
      "rewards/rejected": 0.1091131642460823,
      "step": 19,
      "tokens/chosen_response_len_mean": 263.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 253.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.1003754961271675,
      "learning_rate": 8e-08,
      "logits/policy_chosen": -2.8774542808532715,
      "logits/policy_rejected": -2.6342382431030273,
      "logits/reference_chosen": -2.742877244949341,
      "logits/reference_rejected": -2.4385361671447754,
      "logps/policy_chosen": -104.96978759765625,
      "logps/policy_rejected": -157.80657958984375,
      "logps/reference_chosen": -107.06401062011719,
      "logps/reference_rejected": -159.94366455078125,
      "loss": 0.6774,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02094227820634842,
      "rewards/margins": -0.00042874296195805073,
      "rewards/rejected": 0.02137102000415325,
      "step": 20,
      "tokens/chosen_response_len_mean": 111.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 124.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.113555044323313,
      "learning_rate": 8.4e-08,
      "logits/policy_chosen": -3.1314477920532227,
      "logits/policy_rejected": -2.8576135635375977,
      "logits/reference_chosen": -3.1075565814971924,
      "logits/reference_rejected": -2.6684985160827637,
      "logps/policy_chosen": -121.34175109863281,
      "logps/policy_rejected": -152.86807250976562,
      "logps/reference_chosen": -127.80372619628906,
      "logps/reference_rejected": -158.0865478515625,
      "loss": 0.6994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06461974233388901,
      "rewards/margins": 0.012434981763362885,
      "rewards/rejected": 0.05218476057052612,
      "step": 21,
      "tokens/chosen_response_len_mean": 317.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 203.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.8572997186380196,
      "learning_rate": 8.8e-08,
      "logits/policy_chosen": -3.023026466369629,
      "logits/policy_rejected": -2.970913887023926,
      "logits/reference_chosen": -2.9523911476135254,
      "logits/reference_rejected": -2.8476057052612305,
      "logps/policy_chosen": -123.84272003173828,
      "logps/policy_rejected": -210.03341674804688,
      "logps/reference_chosen": -129.73898315429688,
      "logps/reference_rejected": -216.26409912109375,
      "loss": 0.672,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.05896264314651489,
      "rewards/margins": -0.0033443691208958626,
      "rewards/rejected": 0.06230701133608818,
      "step": 22,
      "tokens/chosen_response_len_mean": 241.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 224.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.4344792354730034,
      "learning_rate": 9.199999999999999e-08,
      "logits/policy_chosen": -2.45721435546875,
      "logits/policy_rejected": -2.4645514488220215,
      "logits/reference_chosen": -2.429760694503784,
      "logits/reference_rejected": -2.448065757751465,
      "logps/policy_chosen": -107.37543487548828,
      "logps/policy_rejected": -120.32754516601562,
      "logps/reference_chosen": -113.06903839111328,
      "logps/reference_rejected": -126.56575012207031,
      "loss": 0.6748,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.056935906410217285,
      "rewards/margins": -0.005446167662739754,
      "rewards/rejected": 0.06238207593560219,
      "step": 23,
      "tokens/chosen_response_len_mean": 314.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 315.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 3.5341818462448504,
      "learning_rate": 9.6e-08,
      "logits/policy_chosen": -2.5186548233032227,
      "logits/policy_rejected": -2.3946735858917236,
      "logits/reference_chosen": -2.5037264823913574,
      "logits/reference_rejected": -2.4204611778259277,
      "logps/policy_chosen": -71.1528549194336,
      "logps/policy_rejected": -96.15353393554688,
      "logps/reference_chosen": -74.45181274414062,
      "logps/reference_rejected": -93.63554382324219,
      "loss": 0.6593,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03298955038189888,
      "rewards/margins": 0.05816952511668205,
      "rewards/rejected": -0.025179967284202576,
      "step": 24,
      "tokens/chosen_response_len_mean": 247.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 155.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.6672688539385514,
      "learning_rate": 1e-07,
      "logits/policy_chosen": -3.052779197692871,
      "logits/policy_rejected": -2.9980740547180176,
      "logits/reference_chosen": -3.0356476306915283,
      "logits/reference_rejected": -2.9704558849334717,
      "logps/policy_chosen": -158.24560546875,
      "logps/policy_rejected": -246.05154418945312,
      "logps/reference_chosen": -166.8022003173828,
      "logps/reference_rejected": -252.77267456054688,
      "loss": 0.6801,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08556582033634186,
      "rewards/margins": 0.01835446059703827,
      "rewards/rejected": 0.06721135973930359,
      "step": 25,
      "tokens/chosen_response_len_mean": 302.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 309.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.157863179356832,
      "learning_rate": 1.0399999999999999e-07,
      "logits/policy_chosen": -2.8244199752807617,
      "logits/policy_rejected": -2.8272688388824463,
      "logits/reference_chosen": -2.8120579719543457,
      "logits/reference_rejected": -2.81868839263916,
      "logps/policy_chosen": -188.75546264648438,
      "logps/policy_rejected": -245.15794372558594,
      "logps/reference_chosen": -201.83450317382812,
      "logps/reference_rejected": -256.3631591796875,
      "loss": 0.6824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13079024851322174,
      "rewards/margins": 0.018737995997071266,
      "rewards/rejected": 0.11205224692821503,
      "step": 26,
      "tokens/chosen_response_len_mean": 499.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 500.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.6785946054818295,
      "learning_rate": 1.0799999999999999e-07,
      "logits/policy_chosen": -3.041701078414917,
      "logits/policy_rejected": -3.0683014392852783,
      "logits/reference_chosen": -3.021995782852173,
      "logits/reference_rejected": -3.0257372856140137,
      "logps/policy_chosen": -102.09321594238281,
      "logps/policy_rejected": -163.72860717773438,
      "logps/reference_chosen": -109.2179946899414,
      "logps/reference_rejected": -167.15447998046875,
      "loss": 0.6755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07124768942594528,
      "rewards/margins": 0.036988936364650726,
      "rewards/rejected": 0.03425874561071396,
      "step": 27,
      "tokens/chosen_response_len_mean": 243.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 237.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 3.0106278359730614,
      "learning_rate": 1.12e-07,
      "logits/policy_chosen": -3.058267116546631,
      "logits/policy_rejected": -3.0097880363464355,
      "logits/reference_chosen": -3.000871181488037,
      "logits/reference_rejected": -2.860621690750122,
      "logps/policy_chosen": -177.05154418945312,
      "logps/policy_rejected": -241.18173217773438,
      "logps/reference_chosen": -187.62625122070312,
      "logps/reference_rejected": -242.83314514160156,
      "loss": 0.6642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10574702173471451,
      "rewards/margins": 0.08923279494047165,
      "rewards/rejected": 0.01651422120630741,
      "step": 28,
      "tokens/chosen_response_len_mean": 391.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 363.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 1.9714081999119137,
      "learning_rate": 1.16e-07,
      "logits/policy_chosen": -2.7036781311035156,
      "logits/policy_rejected": -2.706456422805786,
      "logits/reference_chosen": -2.6821694374084473,
      "logits/reference_rejected": -2.70880126953125,
      "logps/policy_chosen": -50.51161193847656,
      "logps/policy_rejected": -97.51376342773438,
      "logps/reference_chosen": -53.98440170288086,
      "logps/reference_rejected": -100.45523834228516,
      "loss": 0.6839,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03472791612148285,
      "rewards/margins": 0.005313164554536343,
      "rewards/rejected": 0.029414750635623932,
      "step": 29,
      "tokens/chosen_response_len_mean": 246.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 215.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 3.3945073696477333,
      "learning_rate": 1.2e-07,
      "logits/policy_chosen": -2.679271936416626,
      "logits/policy_rejected": -2.747288227081299,
      "logits/reference_chosen": -2.648017406463623,
      "logits/reference_rejected": -2.542226552963257,
      "logps/policy_chosen": -181.09521484375,
      "logps/policy_rejected": -349.05694580078125,
      "logps/reference_chosen": -202.54962158203125,
      "logps/reference_rejected": -355.8513488769531,
      "loss": 0.6558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21454395353794098,
      "rewards/margins": 0.14660006761550903,
      "rewards/rejected": 0.06794389337301254,
      "step": 30,
      "tokens/chosen_response_len_mean": 309.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 268.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.729481181496147,
      "learning_rate": 1.24e-07,
      "logits/policy_chosen": -3.1615757942199707,
      "logits/policy_rejected": -2.618534564971924,
      "logits/reference_chosen": -3.148972511291504,
      "logits/reference_rejected": -2.28782320022583,
      "logps/policy_chosen": -244.06692504882812,
      "logps/policy_rejected": -395.2162780761719,
      "logps/reference_chosen": -254.54090881347656,
      "logps/reference_rejected": -397.2469177246094,
      "loss": 0.6873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10473988205194473,
      "rewards/margins": 0.08443356305360794,
      "rewards/rejected": 0.020306318998336792,
      "step": 31,
      "tokens/chosen_response_len_mean": 410.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.1927971055522133,
      "learning_rate": 1.28e-07,
      "logits/policy_chosen": -2.935199499130249,
      "logits/policy_rejected": -2.862617254257202,
      "logits/reference_chosen": -2.9165947437286377,
      "logits/reference_rejected": -2.809278964996338,
      "logps/policy_chosen": -156.64752197265625,
      "logps/policy_rejected": -224.47006225585938,
      "logps/reference_chosen": -176.1259765625,
      "logps/reference_rejected": -239.154296875,
      "loss": 0.6749,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19478438794612885,
      "rewards/margins": 0.04794217646121979,
      "rewards/rejected": 0.14684221148490906,
      "step": 32,
      "tokens/chosen_response_len_mean": 545.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 527.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.6358653856327257,
      "learning_rate": 1.32e-07,
      "logits/policy_chosen": -2.5262393951416016,
      "logits/policy_rejected": -2.364823341369629,
      "logits/reference_chosen": -2.506678342819214,
      "logits/reference_rejected": -1.6685891151428223,
      "logps/policy_chosen": -104.21517181396484,
      "logps/policy_rejected": -126.63694763183594,
      "logps/reference_chosen": -114.83543395996094,
      "logps/reference_rejected": -136.51809692382812,
      "loss": 0.6837,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10620267689228058,
      "rewards/margins": 0.007391175255179405,
      "rewards/rejected": 0.09881150722503662,
      "step": 33,
      "tokens/chosen_response_len_mean": 448.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 339.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.626247472893481,
      "learning_rate": 1.36e-07,
      "logits/policy_chosen": -3.0164244174957275,
      "logits/policy_rejected": -2.670468330383301,
      "logits/reference_chosen": -2.9480087757110596,
      "logits/reference_rejected": -2.077363967895508,
      "logps/policy_chosen": -144.32318115234375,
      "logps/policy_rejected": -178.09408569335938,
      "logps/reference_chosen": -154.3887939453125,
      "logps/reference_rejected": -182.9420623779297,
      "loss": 0.6748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1006559506058693,
      "rewards/margins": 0.052176106721162796,
      "rewards/rejected": 0.0484798438847065,
      "step": 34,
      "tokens/chosen_response_len_mean": 407.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 217.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.296064376513374,
      "learning_rate": 1.4e-07,
      "logits/policy_chosen": -2.598402738571167,
      "logits/policy_rejected": -2.6598832607269287,
      "logits/reference_chosen": -2.553514003753662,
      "logits/reference_rejected": -2.6580069065093994,
      "logps/policy_chosen": -281.2328186035156,
      "logps/policy_rejected": -179.6129608154297,
      "logps/reference_chosen": -298.13836669921875,
      "logps/reference_rejected": -186.5122528076172,
      "loss": 0.6684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16905567049980164,
      "rewards/margins": 0.10006258636713028,
      "rewards/rejected": 0.06899307668209076,
      "step": 35,
      "tokens/chosen_response_len_mean": 377.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 248.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.898376793238501,
      "learning_rate": 1.44e-07,
      "logits/policy_chosen": -2.7883753776550293,
      "logits/policy_rejected": -2.6082558631896973,
      "logits/reference_chosen": -2.780714273452759,
      "logits/reference_rejected": -2.583864212036133,
      "logps/policy_chosen": -105.63471984863281,
      "logps/policy_rejected": -245.55908203125,
      "logps/reference_chosen": -110.00314331054688,
      "logps/reference_rejected": -244.04986572265625,
      "loss": 0.6715,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04368434101343155,
      "rewards/margins": 0.05877665430307388,
      "rewards/rejected": -0.015092315152287483,
      "step": 36,
      "tokens/chosen_response_len_mean": 183.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 284.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.6935110955386,
      "learning_rate": 1.4799999999999998e-07,
      "logits/policy_chosen": -3.0653228759765625,
      "logits/policy_rejected": -2.882253646850586,
      "logits/reference_chosen": -3.0485243797302246,
      "logits/reference_rejected": -2.6053519248962402,
      "logps/policy_chosen": -186.7164764404297,
      "logps/policy_rejected": -267.58123779296875,
      "logps/reference_chosen": -204.80267333984375,
      "logps/reference_rejected": -272.190185546875,
      "loss": 0.6697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1808619499206543,
      "rewards/margins": 0.13477247953414917,
      "rewards/rejected": 0.046089474111795425,
      "step": 37,
      "tokens/chosen_response_len_mean": 433.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 321.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.597922955352179,
      "learning_rate": 1.5199999999999998e-07,
      "logits/policy_chosen": -2.699632167816162,
      "logits/policy_rejected": -2.7144150733947754,
      "logits/reference_chosen": -2.658318519592285,
      "logits/reference_rejected": -2.691765546798706,
      "logps/policy_chosen": -135.453369140625,
      "logps/policy_rejected": -242.96441650390625,
      "logps/reference_chosen": -146.15704345703125,
      "logps/reference_rejected": -248.74380493164062,
      "loss": 0.6585,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10703684389591217,
      "rewards/margins": 0.049242909997701645,
      "rewards/rejected": 0.057793937623500824,
      "step": 38,
      "tokens/chosen_response_len_mean": 375.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 387.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.079049499920605,
      "learning_rate": 1.56e-07,
      "logits/policy_chosen": -3.077770233154297,
      "logits/policy_rejected": -3.098449945449829,
      "logits/reference_chosen": -3.0287766456604004,
      "logits/reference_rejected": -3.0804648399353027,
      "logps/policy_chosen": -12.474931716918945,
      "logps/policy_rejected": -25.898088455200195,
      "logps/reference_chosen": -12.915287971496582,
      "logps/reference_rejected": -27.526098251342773,
      "loss": 0.6793,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.00440356507897377,
      "rewards/margins": -0.011876537464559078,
      "rewards/rejected": 0.016280101612210274,
      "step": 39,
      "tokens/chosen_response_len_mean": 23.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 25.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.9270192927802197,
      "learning_rate": 1.6e-07,
      "logits/policy_chosen": -2.821016550064087,
      "logits/policy_rejected": -2.7184388637542725,
      "logits/reference_chosen": -2.5926756858825684,
      "logits/reference_rejected": -2.7159690856933594,
      "logps/policy_chosen": -115.9990005493164,
      "logps/policy_rejected": -150.73153686523438,
      "logps/reference_chosen": -120.78485107421875,
      "logps/reference_rejected": -157.8815460205078,
      "loss": 0.668,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04785853251814842,
      "rewards/margins": -0.02364155277609825,
      "rewards/rejected": 0.07150008529424667,
      "step": 40,
      "tokens/chosen_response_len_mean": 179.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 168.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.807071363383845,
      "learning_rate": 1.64e-07,
      "logits/policy_chosen": -2.6167349815368652,
      "logits/policy_rejected": -2.5763139724731445,
      "logits/reference_chosen": -2.513126850128174,
      "logits/reference_rejected": -2.5481503009796143,
      "logps/policy_chosen": -213.66064453125,
      "logps/policy_rejected": -274.35003662109375,
      "logps/reference_chosen": -240.58090209960938,
      "logps/reference_rejected": -297.9310607910156,
      "loss": 0.6847,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.26920264959335327,
      "rewards/margins": 0.03339245170354843,
      "rewards/rejected": 0.23581019043922424,
      "step": 41,
      "tokens/chosen_response_len_mean": 468.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 442.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 3.1342954098759788,
      "learning_rate": 1.68e-07,
      "logits/policy_chosen": -2.98238205909729,
      "logits/policy_rejected": -2.6742758750915527,
      "logits/reference_chosen": -2.9786105155944824,
      "logits/reference_rejected": -2.5011706352233887,
      "logps/policy_chosen": -400.793701171875,
      "logps/policy_rejected": -471.21826171875,
      "logps/reference_chosen": -424.1178283691406,
      "logps/reference_rejected": -484.6050720214844,
      "loss": 0.6671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23324130475521088,
      "rewards/margins": 0.09937307238578796,
      "rewards/rejected": 0.1338682323694229,
      "step": 42,
      "tokens/chosen_response_len_mean": 593.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 479.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 1.7422493551589444,
      "learning_rate": 1.7199999999999998e-07,
      "logits/policy_chosen": -2.763662338256836,
      "logits/policy_rejected": -2.434535503387451,
      "logits/reference_chosen": -2.710179567337036,
      "logits/reference_rejected": -1.868179440498352,
      "logps/policy_chosen": -77.93124389648438,
      "logps/policy_rejected": -110.21360778808594,
      "logps/reference_chosen": -86.2520751953125,
      "logps/reference_rejected": -110.11550903320312,
      "loss": 0.679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08320826292037964,
      "rewards/margins": 0.08418922126293182,
      "rewards/rejected": -0.0009809592738747597,
      "step": 43,
      "tokens/chosen_response_len_mean": 152.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 85.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 3.138281711904061,
      "learning_rate": 1.76e-07,
      "logits/policy_chosen": -2.8771750926971436,
      "logits/policy_rejected": -2.9175660610198975,
      "logits/reference_chosen": -2.8088254928588867,
      "logits/reference_rejected": -2.877462863922119,
      "logps/policy_chosen": -37.14703369140625,
      "logps/policy_rejected": -55.703121185302734,
      "logps/reference_chosen": -38.25192642211914,
      "logps/reference_rejected": -56.50023651123047,
      "loss": 0.686,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011048920452594757,
      "rewards/margins": 0.0030777817592024803,
      "rewards/rejected": 0.007971138693392277,
      "step": 44,
      "tokens/chosen_response_len_mean": 66.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 70.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 3.284990775967774,
      "learning_rate": 1.8e-07,
      "logits/policy_chosen": -2.8581864833831787,
      "logits/policy_rejected": -2.766713857650757,
      "logits/reference_chosen": -2.8522939682006836,
      "logits/reference_rejected": -2.7666430473327637,
      "logps/policy_chosen": -165.7818603515625,
      "logps/policy_rejected": -241.84002685546875,
      "logps/reference_chosen": -181.072998046875,
      "logps/reference_rejected": -250.3277587890625,
      "loss": 0.6537,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15291133522987366,
      "rewards/margins": 0.06803382933139801,
      "rewards/rejected": 0.08487750589847565,
      "step": 45,
      "tokens/chosen_response_len_mean": 560.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 473.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.463717582717747,
      "learning_rate": 1.8399999999999998e-07,
      "logits/policy_chosen": -2.900191068649292,
      "logits/policy_rejected": -2.9184107780456543,
      "logits/reference_chosen": -2.8673532009124756,
      "logits/reference_rejected": -2.892207145690918,
      "logps/policy_chosen": -51.21653747558594,
      "logps/policy_rejected": -107.85240936279297,
      "logps/reference_chosen": -54.76094436645508,
      "logps/reference_rejected": -117.80521392822266,
      "loss": 0.695,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.035444095730781555,
      "rewards/margins": -0.06408397108316422,
      "rewards/rejected": 0.09952806681394577,
      "step": 46,
      "tokens/chosen_response_len_mean": 109.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 108.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 1.9738986795582658,
      "learning_rate": 1.88e-07,
      "logits/policy_chosen": -2.2502527236938477,
      "logits/policy_rejected": -2.245668649673462,
      "logits/reference_chosen": -2.177823781967163,
      "logits/reference_rejected": -2.223409652709961,
      "logps/policy_chosen": -37.820743560791016,
      "logps/policy_rejected": -99.87039184570312,
      "logps/reference_chosen": -44.7762451171875,
      "logps/reference_rejected": -104.45420837402344,
      "loss": 0.6825,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06955502927303314,
      "rewards/margins": 0.023716777563095093,
      "rewards/rejected": 0.04583824798464775,
      "step": 47,
      "tokens/chosen_response_len_mean": 93.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 110.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.8813658188320335,
      "learning_rate": 1.92e-07,
      "logits/policy_chosen": -2.6205151081085205,
      "logits/policy_rejected": -2.5673413276672363,
      "logits/reference_chosen": -2.5005288124084473,
      "logits/reference_rejected": -2.561748504638672,
      "logps/policy_chosen": -65.36790466308594,
      "logps/policy_rejected": -167.23562622070312,
      "logps/reference_chosen": -72.64520263671875,
      "logps/reference_rejected": -172.6849365234375,
      "loss": 0.6706,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.07277299463748932,
      "rewards/margins": 0.018280120566487312,
      "rewards/rejected": 0.054492875933647156,
      "step": 48,
      "tokens/chosen_response_len_mean": 199.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 209.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.862318181422901,
      "learning_rate": 1.96e-07,
      "logits/policy_chosen": -2.701411247253418,
      "logits/policy_rejected": -2.8611505031585693,
      "logits/reference_chosen": -2.2373385429382324,
      "logits/reference_rejected": -2.845200777053833,
      "logps/policy_chosen": -126.9096908569336,
      "logps/policy_rejected": -269.68243408203125,
      "logps/reference_chosen": -136.80958557128906,
      "logps/reference_rejected": -277.586181640625,
      "loss": 0.6796,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09899893403053284,
      "rewards/margins": 0.01996154710650444,
      "rewards/rejected": 0.0790373906493187,
      "step": 49,
      "tokens/chosen_response_len_mean": 262.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 337.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.324007938345189,
      "learning_rate": 2e-07,
      "logits/policy_chosen": -2.2418417930603027,
      "logits/policy_rejected": -2.1697628498077393,
      "logits/reference_chosen": -2.142324924468994,
      "logits/reference_rejected": -2.1179537773132324,
      "logps/policy_chosen": -136.041259765625,
      "logps/policy_rejected": -266.0906677246094,
      "logps/reference_chosen": -151.6305694580078,
      "logps/reference_rejected": -273.1885986328125,
      "loss": 0.6677,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15589313209056854,
      "rewards/margins": 0.08491363376379013,
      "rewards/rejected": 0.07097949832677841,
      "step": 50,
      "tokens/chosen_response_len_mean": 526.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 469.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.3504496124001744,
      "learning_rate": 2.0399999999999997e-07,
      "logits/policy_chosen": -2.899348497390747,
      "logits/policy_rejected": -2.8138720989227295,
      "logits/reference_chosen": -2.8722875118255615,
      "logits/reference_rejected": -2.8029227256774902,
      "logps/policy_chosen": -63.48197555541992,
      "logps/policy_rejected": -101.47474670410156,
      "logps/reference_chosen": -68.96565246582031,
      "logps/reference_rejected": -108.34456634521484,
      "loss": 0.6831,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.05483681336045265,
      "rewards/margins": -0.013861382380127907,
      "rewards/rejected": 0.06869819760322571,
      "step": 51,
      "tokens/chosen_response_len_mean": 212.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 181.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.4664067603984665,
      "learning_rate": 2.0799999999999998e-07,
      "logits/policy_chosen": -2.710914134979248,
      "logits/policy_rejected": -2.6489503383636475,
      "logits/reference_chosen": -2.6887543201446533,
      "logits/reference_rejected": -2.565074920654297,
      "logps/policy_chosen": -120.6403579711914,
      "logps/policy_rejected": -180.31842041015625,
      "logps/reference_chosen": -129.82321166992188,
      "logps/reference_rejected": -187.31112670898438,
      "loss": 0.6824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0918285921216011,
      "rewards/margins": 0.021901624277234077,
      "rewards/rejected": 0.06992696225643158,
      "step": 52,
      "tokens/chosen_response_len_mean": 340.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 3.7734473565962108,
      "learning_rate": 2.12e-07,
      "logits/policy_chosen": -2.9076874256134033,
      "logits/policy_rejected": -2.8471598625183105,
      "logits/reference_chosen": -2.837496280670166,
      "logits/reference_rejected": -2.783947467803955,
      "logps/policy_chosen": -88.39242553710938,
      "logps/policy_rejected": -232.25318908691406,
      "logps/reference_chosen": -96.45068359375,
      "logps/reference_rejected": -237.35009765625,
      "loss": 0.6784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08058254420757294,
      "rewards/margins": 0.02961355820298195,
      "rewards/rejected": 0.05096898972988129,
      "step": 53,
      "tokens/chosen_response_len_mean": 195.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.8319761915445367,
      "learning_rate": 2.1599999999999998e-07,
      "logits/policy_chosen": -3.144252300262451,
      "logits/policy_rejected": -3.0141947269439697,
      "logits/reference_chosen": -3.116751194000244,
      "logits/reference_rejected": -2.8330681324005127,
      "logps/policy_chosen": -132.089111328125,
      "logps/policy_rejected": -225.1402587890625,
      "logps/reference_chosen": -136.70138549804688,
      "logps/reference_rejected": -224.64724731445312,
      "loss": 0.6737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.046122774481773376,
      "rewards/margins": 0.05105280876159668,
      "rewards/rejected": -0.004930033814162016,
      "step": 54,
      "tokens/chosen_response_len_mean": 299.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 257.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.947847201514034,
      "learning_rate": 2.1999999999999998e-07,
      "logits/policy_chosen": -2.8887228965759277,
      "logits/policy_rejected": -2.68259596824646,
      "logits/reference_chosen": -2.864752769470215,
      "logits/reference_rejected": -2.628319263458252,
      "logps/policy_chosen": -135.53025817871094,
      "logps/policy_rejected": -133.9132843017578,
      "logps/reference_chosen": -145.9614715576172,
      "logps/reference_rejected": -137.02508544921875,
      "loss": 0.666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10431211441755295,
      "rewards/margins": 0.07319390773773193,
      "rewards/rejected": 0.031118206679821014,
      "step": 55,
      "tokens/chosen_response_len_mean": 349.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 196.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.771906040366205,
      "learning_rate": 2.24e-07,
      "logits/policy_chosen": -2.231527328491211,
      "logits/policy_rejected": -2.0439558029174805,
      "logits/reference_chosen": -2.230081796646118,
      "logits/reference_rejected": -2.080432176589966,
      "logps/policy_chosen": -399.80303955078125,
      "logps/policy_rejected": -446.4791259765625,
      "logps/reference_chosen": -431.472412109375,
      "logps/reference_rejected": -470.2456970214844,
      "loss": 0.6709,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.31669357419013977,
      "rewards/margins": 0.07902783155441284,
      "rewards/rejected": 0.23766574263572693,
      "step": 56,
      "tokens/chosen_response_len_mean": 805.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 711.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 1.7428598517814506,
      "learning_rate": 2.28e-07,
      "logits/policy_chosen": -2.7684319019317627,
      "logits/policy_rejected": -2.71665096282959,
      "logits/reference_chosen": -2.747512102127075,
      "logits/reference_rejected": -2.690124034881592,
      "logps/policy_chosen": -85.4046630859375,
      "logps/policy_rejected": -111.96269226074219,
      "logps/reference_chosen": -96.96040344238281,
      "logps/reference_rejected": -121.33839416503906,
      "loss": 0.6888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11555740237236023,
      "rewards/margins": 0.021800296381115913,
      "rewards/rejected": 0.09375710785388947,
      "step": 57,
      "tokens/chosen_response_len_mean": 261.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 248.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.6040542171359817,
      "learning_rate": 2.32e-07,
      "logits/policy_chosen": -2.989000082015991,
      "logits/policy_rejected": -2.8631539344787598,
      "logits/reference_chosen": -2.9670627117156982,
      "logits/reference_rejected": -2.5129096508026123,
      "logps/policy_chosen": -87.29337310791016,
      "logps/policy_rejected": -119.82539367675781,
      "logps/reference_chosen": -95.52102661132812,
      "logps/reference_rejected": -122.27404022216797,
      "loss": 0.6657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08227652311325073,
      "rewards/margins": 0.05779005587100983,
      "rewards/rejected": 0.024486469104886055,
      "step": 58,
      "tokens/chosen_response_len_mean": 162.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 138.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 3.249420407772776,
      "learning_rate": 2.3599999999999997e-07,
      "logits/policy_chosen": -2.9996423721313477,
      "logits/policy_rejected": -2.974095582962036,
      "logits/reference_chosen": -2.964367151260376,
      "logits/reference_rejected": -2.953282356262207,
      "logps/policy_chosen": -56.97121810913086,
      "logps/policy_rejected": -141.20883178710938,
      "logps/reference_chosen": -58.637413024902344,
      "logps/reference_rejected": -143.08058166503906,
      "loss": 0.679,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0166619960218668,
      "rewards/margins": -0.0020555637311190367,
      "rewards/rejected": 0.018717560917139053,
      "step": 59,
      "tokens/chosen_response_len_mean": 121.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 127.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 3.503891824715986,
      "learning_rate": 2.4e-07,
      "logits/policy_chosen": -3.207751512527466,
      "logits/policy_rejected": -2.8613648414611816,
      "logits/reference_chosen": -3.1973516941070557,
      "logits/reference_rejected": -2.244464874267578,
      "logps/policy_chosen": -115.4853286743164,
      "logps/policy_rejected": -205.64161682128906,
      "logps/reference_chosen": -123.15602111816406,
      "logps/reference_rejected": -206.07244873046875,
      "loss": 0.6713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0767069011926651,
      "rewards/margins": 0.07239863276481628,
      "rewards/rejected": 0.004308272153139114,
      "step": 60,
      "tokens/chosen_response_len_mean": 262.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 185.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.987682806191956,
      "learning_rate": 2.4399999999999996e-07,
      "logits/policy_chosen": -2.65006947517395,
      "logits/policy_rejected": -2.60097599029541,
      "logits/reference_chosen": -2.6217522621154785,
      "logits/reference_rejected": -2.507542610168457,
      "logps/policy_chosen": -184.2285614013672,
      "logps/policy_rejected": -184.65708923339844,
      "logps/reference_chosen": -215.3209228515625,
      "logps/reference_rejected": -187.43243408203125,
      "loss": 0.6592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31092363595962524,
      "rewards/margins": 0.2831703722476959,
      "rewards/rejected": 0.027753248810768127,
      "step": 61,
      "tokens/chosen_response_len_mean": 344.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 292.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 3.02187794129979,
      "learning_rate": 2.48e-07,
      "logits/policy_chosen": -3.102184534072876,
      "logits/policy_rejected": -3.0169076919555664,
      "logits/reference_chosen": -3.0980794429779053,
      "logits/reference_rejected": -3.06693434715271,
      "logps/policy_chosen": -180.20120239257812,
      "logps/policy_rejected": -290.76556396484375,
      "logps/reference_chosen": -186.42239379882812,
      "logps/reference_rejected": -299.5263977050781,
      "loss": 0.6663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06221204251050949,
      "rewards/margins": -0.025396330282092094,
      "rewards/rejected": 0.08760837465524673,
      "step": 62,
      "tokens/chosen_response_len_mean": 628.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 573.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.2026851566699364,
      "learning_rate": 2.52e-07,
      "logits/policy_chosen": -2.9184327125549316,
      "logits/policy_rejected": -2.7862141132354736,
      "logits/reference_chosen": -2.9029111862182617,
      "logits/reference_rejected": -2.562068462371826,
      "logps/policy_chosen": -176.180908203125,
      "logps/policy_rejected": -224.06822204589844,
      "logps/reference_chosen": -191.051025390625,
      "logps/reference_rejected": -231.65249633789062,
      "loss": 0.682,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14870098233222961,
      "rewards/margins": 0.07285812497138977,
      "rewards/rejected": 0.07584285736083984,
      "step": 63,
      "tokens/chosen_response_len_mean": 383.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 315.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.6923985748881183,
      "learning_rate": 2.56e-07,
      "logits/policy_chosen": -3.1094112396240234,
      "logits/policy_rejected": -3.059873104095459,
      "logits/reference_chosen": -3.0696709156036377,
      "logits/reference_rejected": -3.0277552604675293,
      "logps/policy_chosen": -109.50044250488281,
      "logps/policy_rejected": -227.6053009033203,
      "logps/reference_chosen": -114.45265197753906,
      "logps/reference_rejected": -232.44473266601562,
      "loss": 0.6689,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04952218383550644,
      "rewards/margins": 0.0011278600431978703,
      "rewards/rejected": 0.04839432239532471,
      "step": 64,
      "tokens/chosen_response_len_mean": 172.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 173.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.3372174586928685,
      "learning_rate": 2.6e-07,
      "logits/policy_chosen": -2.9087581634521484,
      "logits/policy_rejected": -2.973167657852173,
      "logits/reference_chosen": -2.754256248474121,
      "logits/reference_rejected": -2.943002223968506,
      "logps/policy_chosen": -16.885406494140625,
      "logps/policy_rejected": -58.0799560546875,
      "logps/reference_chosen": -16.55327033996582,
      "logps/reference_rejected": -58.852867126464844,
      "loss": 0.6709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0033213470596820116,
      "rewards/margins": -0.011050481349229813,
      "rewards/rejected": 0.007729134988039732,
      "step": 65,
      "tokens/chosen_response_len_mean": 25.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 37.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 1.8439881768995727,
      "learning_rate": 2.64e-07,
      "logits/policy_chosen": -3.2132272720336914,
      "logits/policy_rejected": -3.156252861022949,
      "logits/reference_chosen": -3.1969757080078125,
      "logits/reference_rejected": -3.008265733718872,
      "logps/policy_chosen": -89.50849914550781,
      "logps/policy_rejected": -105.58460998535156,
      "logps/reference_chosen": -96.482421875,
      "logps/reference_rejected": -111.33866119384766,
      "loss": 0.6855,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06973925232887268,
      "rewards/margins": 0.0121987946331501,
      "rewards/rejected": 0.05754045769572258,
      "step": 66,
      "tokens/chosen_response_len_mean": 163.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 142.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 1.9679864280666897,
      "learning_rate": 2.68e-07,
      "logits/policy_chosen": -2.443567991256714,
      "logits/policy_rejected": -2.2462878227233887,
      "logits/reference_chosen": -2.3940353393554688,
      "logits/reference_rejected": -1.4334111213684082,
      "logps/policy_chosen": -133.6657257080078,
      "logps/policy_rejected": -126.65130615234375,
      "logps/reference_chosen": -143.77218627929688,
      "logps/reference_rejected": -130.96572875976562,
      "loss": 0.6801,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.10106448829174042,
      "rewards/margins": 0.057920388877391815,
      "rewards/rejected": 0.0431441031396389,
      "step": 67,
      "tokens/chosen_response_len_mean": 360.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 238.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.3444172227352817,
      "learning_rate": 2.72e-07,
      "logits/policy_chosen": -2.447854518890381,
      "logits/policy_rejected": -2.4554026126861572,
      "logits/reference_chosen": -2.4224636554718018,
      "logits/reference_rejected": -2.44649600982666,
      "logps/policy_chosen": -116.48789978027344,
      "logps/policy_rejected": -196.64881896972656,
      "logps/reference_chosen": -129.35415649414062,
      "logps/reference_rejected": -209.97300720214844,
      "loss": 0.6714,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12866270542144775,
      "rewards/margins": -0.004579143598675728,
      "rewards/rejected": 0.13324186205863953,
      "step": 68,
      "tokens/chosen_response_len_mean": 627.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 597.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 1.9355500776493433,
      "learning_rate": 2.7600000000000004e-07,
      "logits/policy_chosen": -3.001335620880127,
      "logits/policy_rejected": -2.8170900344848633,
      "logits/reference_chosen": -2.9873673915863037,
      "logits/reference_rejected": -2.8246562480926514,
      "logps/policy_chosen": -122.1274642944336,
      "logps/policy_rejected": -174.14083862304688,
      "logps/reference_chosen": -129.572509765625,
      "logps/reference_rejected": -175.02163696289062,
      "loss": 0.6814,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07445055991411209,
      "rewards/margins": 0.06564265489578247,
      "rewards/rejected": 0.008807908743619919,
      "step": 69,
      "tokens/chosen_response_len_mean": 316.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 212.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 3.1880887553688613,
      "learning_rate": 2.8e-07,
      "logits/policy_chosen": -2.666834831237793,
      "logits/policy_rejected": -2.602752208709717,
      "logits/reference_chosen": -2.5575764179229736,
      "logits/reference_rejected": -2.594496965408325,
      "logps/policy_chosen": -144.6117401123047,
      "logps/policy_rejected": -323.0421447753906,
      "logps/reference_chosen": -156.17013549804688,
      "logps/reference_rejected": -333.77972412109375,
      "loss": 0.6725,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11558418720960617,
      "rewards/margins": 0.00820855051279068,
      "rewards/rejected": 0.10737563669681549,
      "step": 70,
      "tokens/chosen_response_len_mean": 319.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 349.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.476324701775898,
      "learning_rate": 2.8399999999999995e-07,
      "logits/policy_chosen": -2.8258767127990723,
      "logits/policy_rejected": -2.808089017868042,
      "logits/reference_chosen": -2.8227195739746094,
      "logits/reference_rejected": -2.8310012817382812,
      "logps/policy_chosen": -141.63214111328125,
      "logps/policy_rejected": -178.0096893310547,
      "logps/reference_chosen": -151.22640991210938,
      "logps/reference_rejected": -182.14822387695312,
      "loss": 0.6621,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09594269841909409,
      "rewards/margins": 0.054557494819164276,
      "rewards/rejected": 0.04138519987463951,
      "step": 71,
      "tokens/chosen_response_len_mean": 486.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 302.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.699352211419919,
      "learning_rate": 2.88e-07,
      "logits/policy_chosen": -2.597538709640503,
      "logits/policy_rejected": -2.41672945022583,
      "logits/reference_chosen": -2.5760765075683594,
      "logits/reference_rejected": -1.9195555448532104,
      "logps/policy_chosen": -247.6851043701172,
      "logps/policy_rejected": -382.4797058105469,
      "logps/reference_chosen": -263.0463562011719,
      "logps/reference_rejected": -387.8023376464844,
      "loss": 0.6657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15361259877681732,
      "rewards/margins": 0.10038647800683975,
      "rewards/rejected": 0.05322612449526787,
      "step": 72,
      "tokens/chosen_response_len_mean": 411.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 344.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.5969468603738255,
      "learning_rate": 2.9199999999999997e-07,
      "logits/policy_chosen": -2.7702274322509766,
      "logits/policy_rejected": -2.5614566802978516,
      "logits/reference_chosen": -2.7665748596191406,
      "logits/reference_rejected": -2.3978798389434814,
      "logps/policy_chosen": -72.09918212890625,
      "logps/policy_rejected": -68.87893676757812,
      "logps/reference_chosen": -80.68479919433594,
      "logps/reference_rejected": -67.986328125,
      "loss": 0.6635,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08585613965988159,
      "rewards/margins": 0.0947822630405426,
      "rewards/rejected": -0.008926120586693287,
      "step": 73,
      "tokens/chosen_response_len_mean": 205.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 90.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.8994112666814575,
      "learning_rate": 2.9599999999999995e-07,
      "logits/policy_chosen": -2.9491050243377686,
      "logits/policy_rejected": -2.9374337196350098,
      "logits/reference_chosen": -2.924247980117798,
      "logits/reference_rejected": -2.8732385635375977,
      "logps/policy_chosen": -163.5004425048828,
      "logps/policy_rejected": -186.22500610351562,
      "logps/reference_chosen": -180.18331909179688,
      "logps/reference_rejected": -202.66912841796875,
      "loss": 0.6673,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.16682888567447662,
      "rewards/margins": 0.00238775834441185,
      "rewards/rejected": 0.16444112360477448,
      "step": 74,
      "tokens/chosen_response_len_mean": 365.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 331.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.1648523975476306,
      "learning_rate": 3e-07,
      "logits/policy_chosen": -2.1604349613189697,
      "logits/policy_rejected": -2.168158769607544,
      "logits/reference_chosen": -2.1622910499572754,
      "logits/reference_rejected": -2.1353628635406494,
      "logps/policy_chosen": -88.18841552734375,
      "logps/policy_rejected": -119.30603790283203,
      "logps/reference_chosen": -93.11495971679688,
      "logps/reference_rejected": -123.05696868896484,
      "loss": 0.6881,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04926542192697525,
      "rewards/margins": 0.011756086722016335,
      "rewards/rejected": 0.037509337067604065,
      "step": 75,
      "tokens/chosen_response_len_mean": 258.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 199.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.403809139260171,
      "learning_rate": 3.0399999999999997e-07,
      "logits/policy_chosen": -2.830843925476074,
      "logits/policy_rejected": -2.8032288551330566,
      "logits/reference_chosen": -2.8209171295166016,
      "logits/reference_rejected": -2.797828435897827,
      "logps/policy_chosen": -69.43246459960938,
      "logps/policy_rejected": -162.20603942871094,
      "logps/reference_chosen": -74.48234558105469,
      "logps/reference_rejected": -164.12376403808594,
      "loss": 0.6658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05049885809421539,
      "rewards/margins": 0.03132178634405136,
      "rewards/rejected": 0.019177071750164032,
      "step": 76,
      "tokens/chosen_response_len_mean": 142.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 126.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.854272678012172,
      "learning_rate": 3.08e-07,
      "logits/policy_chosen": -3.0018627643585205,
      "logits/policy_rejected": -2.6590089797973633,
      "logits/reference_chosen": -2.995105028152466,
      "logits/reference_rejected": -2.385610580444336,
      "logps/policy_chosen": -85.27000427246094,
      "logps/policy_rejected": -132.92564392089844,
      "logps/reference_chosen": -90.3302001953125,
      "logps/reference_rejected": -134.35781860351562,
      "loss": 0.6608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05060192570090294,
      "rewards/margins": 0.03628016635775566,
      "rewards/rejected": 0.014321764931082726,
      "step": 77,
      "tokens/chosen_response_len_mean": 363.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 320.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.7505136356983133,
      "learning_rate": 3.12e-07,
      "logits/policy_chosen": -2.0338382720947266,
      "logits/policy_rejected": -2.276106357574463,
      "logits/reference_chosen": -2.0206451416015625,
      "logits/reference_rejected": -2.060913562774658,
      "logps/policy_chosen": -232.1044464111328,
      "logps/policy_rejected": -146.2607879638672,
      "logps/reference_chosen": -250.27952575683594,
      "logps/reference_rejected": -149.79270935058594,
      "loss": 0.6679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18175065517425537,
      "rewards/margins": 0.14643150568008423,
      "rewards/rejected": 0.03531915321946144,
      "step": 78,
      "tokens/chosen_response_len_mean": 521.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 307.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.4952987336105665,
      "learning_rate": 3.1599999999999997e-07,
      "logits/policy_chosen": -2.8599331378936768,
      "logits/policy_rejected": -2.8679845333099365,
      "logits/reference_chosen": -2.7079713344573975,
      "logits/reference_rejected": -2.4389915466308594,
      "logps/policy_chosen": -148.4790496826172,
      "logps/policy_rejected": -277.09405517578125,
      "logps/reference_chosen": -156.32412719726562,
      "logps/reference_rejected": -281.2509765625,
      "loss": 0.6709,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0784510001540184,
      "rewards/margins": 0.03688175603747368,
      "rewards/rejected": 0.041569244116544724,
      "step": 79,
      "tokens/chosen_response_len_mean": 276.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 336.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.3479129695327297,
      "learning_rate": 3.2e-07,
      "logits/policy_chosen": -1.8025144338607788,
      "logits/policy_rejected": -1.9184333086013794,
      "logits/reference_chosen": -1.7402411699295044,
      "logits/reference_rejected": -1.906538724899292,
      "logps/policy_chosen": -79.56036376953125,
      "logps/policy_rejected": -185.89906311035156,
      "logps/reference_chosen": -88.7896499633789,
      "logps/reference_rejected": -198.23004150390625,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09229286015033722,
      "rewards/margins": -0.03101687878370285,
      "rewards/rejected": 0.12330973893404007,
      "step": 80,
      "tokens/chosen_response_len_mean": 432.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 410.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 1.4788277018262954,
      "learning_rate": 3.24e-07,
      "logits/policy_chosen": -2.7751684188842773,
      "logits/policy_rejected": -2.7188775539398193,
      "logits/reference_chosen": -2.774470567703247,
      "logits/reference_rejected": -2.3951942920684814,
      "logps/policy_chosen": -64.14806365966797,
      "logps/policy_rejected": -91.78968048095703,
      "logps/reference_chosen": -70.32818603515625,
      "logps/reference_rejected": -92.31802368164062,
      "loss": 0.6813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06180126219987869,
      "rewards/margins": 0.05651785433292389,
      "rewards/rejected": 0.005283408332616091,
      "step": 81,
      "tokens/chosen_response_len_mean": 345.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 316.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.8935594664890503,
      "learning_rate": 3.28e-07,
      "logits/policy_chosen": -2.936018943786621,
      "logits/policy_rejected": -2.9313597679138184,
      "logits/reference_chosen": -2.9295578002929688,
      "logits/reference_rejected": -2.9169669151306152,
      "logps/policy_chosen": -141.27691650390625,
      "logps/policy_rejected": -264.863037109375,
      "logps/reference_chosen": -152.80865478515625,
      "logps/reference_rejected": -274.9104919433594,
      "loss": 0.6644,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11531716585159302,
      "rewards/margins": 0.014842674136161804,
      "rewards/rejected": 0.10047448426485062,
      "step": 82,
      "tokens/chosen_response_len_mean": 394.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 390.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.4649587565099678,
      "learning_rate": 3.32e-07,
      "logits/policy_chosen": -2.9249117374420166,
      "logits/policy_rejected": -2.9090662002563477,
      "logits/reference_chosen": -2.868709087371826,
      "logits/reference_rejected": -2.898679256439209,
      "logps/policy_chosen": -154.240234375,
      "logps/policy_rejected": -297.08160400390625,
      "logps/reference_chosen": -161.24609375,
      "logps/reference_rejected": -297.5753479003906,
      "loss": 0.6687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07005855441093445,
      "rewards/margins": 0.06512109935283661,
      "rewards/rejected": 0.004937455989420414,
      "step": 83,
      "tokens/chosen_response_len_mean": 334.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 309.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.5355636198848175,
      "learning_rate": 3.36e-07,
      "logits/policy_chosen": -2.9488754272460938,
      "logits/policy_rejected": -2.9289844036102295,
      "logits/reference_chosen": -2.890922784805298,
      "logits/reference_rejected": -2.870727062225342,
      "logps/policy_chosen": -18.58037757873535,
      "logps/policy_rejected": -47.89413833618164,
      "logps/reference_chosen": -19.83108901977539,
      "logps/reference_rejected": -48.526878356933594,
      "loss": 0.6755,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012507104314863682,
      "rewards/margins": 0.006179675459861755,
      "rewards/rejected": 0.006327427923679352,
      "step": 84,
      "tokens/chosen_response_len_mean": 27.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 32.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.5319776077967746,
      "learning_rate": 3.4000000000000003e-07,
      "logits/policy_chosen": -2.7499167919158936,
      "logits/policy_rejected": -2.690596103668213,
      "logits/reference_chosen": -2.73750638961792,
      "logits/reference_rejected": -2.6311354637145996,
      "logps/policy_chosen": -123.83195495605469,
      "logps/policy_rejected": -194.00137329101562,
      "logps/reference_chosen": -139.81263732910156,
      "logps/reference_rejected": -196.44784545898438,
      "loss": 0.6523,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.15980683267116547,
      "rewards/margins": 0.13534212112426758,
      "rewards/rejected": 0.024464718997478485,
      "step": 85,
      "tokens/chosen_response_len_mean": 606.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 519.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.007472503017741,
      "learning_rate": 3.4399999999999996e-07,
      "logits/policy_chosen": -2.9675540924072266,
      "logits/policy_rejected": -2.9613142013549805,
      "logits/reference_chosen": -2.9432828426361084,
      "logits/reference_rejected": -2.9374613761901855,
      "logps/policy_chosen": -12.885274887084961,
      "logps/policy_rejected": -34.69200134277344,
      "logps/reference_chosen": -13.591547012329102,
      "logps/reference_rejected": -35.15093994140625,
      "loss": 0.6775,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007062721066176891,
      "rewards/margins": 0.002473349217325449,
      "rewards/rejected": 0.004589371383190155,
      "step": 86,
      "tokens/chosen_response_len_mean": 11.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 12.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.3366851120609398,
      "learning_rate": 3.4799999999999994e-07,
      "logits/policy_chosen": -2.510186195373535,
      "logits/policy_rejected": -2.3673720359802246,
      "logits/reference_chosen": -2.5208425521850586,
      "logits/reference_rejected": -2.307051658630371,
      "logps/policy_chosen": -103.87987518310547,
      "logps/policy_rejected": -118.88311767578125,
      "logps/reference_chosen": -114.24214172363281,
      "logps/reference_rejected": -118.43659973144531,
      "loss": 0.664,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10362273454666138,
      "rewards/margins": 0.10808798670768738,
      "rewards/rejected": -0.004465251229703426,
      "step": 87,
      "tokens/chosen_response_len_mean": 216.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 106.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.4579695955138874,
      "learning_rate": 3.52e-07,
      "logits/policy_chosen": -2.9871153831481934,
      "logits/policy_rejected": -2.8764257431030273,
      "logits/reference_chosen": -2.9515204429626465,
      "logits/reference_rejected": -2.883835792541504,
      "logps/policy_chosen": -151.91807556152344,
      "logps/policy_rejected": -141.33587646484375,
      "logps/reference_chosen": -162.7379608154297,
      "logps/reference_rejected": -147.37442016601562,
      "loss": 0.6695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10819891095161438,
      "rewards/margins": 0.04781348258256912,
      "rewards/rejected": 0.06038542836904526,
      "step": 88,
      "tokens/chosen_response_len_mean": 299.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 223.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.2124030442195703,
      "learning_rate": 3.5599999999999996e-07,
      "logits/policy_chosen": -3.097362995147705,
      "logits/policy_rejected": -2.821688652038574,
      "logits/reference_chosen": -3.0792651176452637,
      "logits/reference_rejected": -2.6497914791107178,
      "logps/policy_chosen": -106.85523986816406,
      "logps/policy_rejected": -142.70628356933594,
      "logps/reference_chosen": -114.98326873779297,
      "logps/reference_rejected": -145.11463928222656,
      "loss": 0.6769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08128029853105545,
      "rewards/margins": 0.05719655007123947,
      "rewards/rejected": 0.02408374845981598,
      "step": 89,
      "tokens/chosen_response_len_mean": 288.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 241.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.726548509097342,
      "learning_rate": 3.6e-07,
      "logits/policy_chosen": -2.911972761154175,
      "logits/policy_rejected": -2.7545974254608154,
      "logits/reference_chosen": -2.8914873600006104,
      "logits/reference_rejected": -2.4686713218688965,
      "logps/policy_chosen": -90.4947509765625,
      "logps/policy_rejected": -144.89512634277344,
      "logps/reference_chosen": -100.72172546386719,
      "logps/reference_rejected": -144.84951782226562,
      "loss": 0.6578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1022697389125824,
      "rewards/margins": 0.10272593796253204,
      "rewards/rejected": -0.0004562041722238064,
      "step": 90,
      "tokens/chosen_response_len_mean": 254.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 112.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.2900988215150644,
      "learning_rate": 3.64e-07,
      "logits/policy_chosen": -2.8383259773254395,
      "logits/policy_rejected": -2.824490547180176,
      "logits/reference_chosen": -2.8273422718048096,
      "logits/reference_rejected": -2.8102381229400635,
      "logps/policy_chosen": -132.3096466064453,
      "logps/policy_rejected": -264.598388671875,
      "logps/reference_chosen": -139.56736755371094,
      "logps/reference_rejected": -264.87725830078125,
      "loss": 0.6825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07257718592882156,
      "rewards/margins": 0.0697886273264885,
      "rewards/rejected": 0.002788563258945942,
      "step": 91,
      "tokens/chosen_response_len_mean": 395.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 397.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.8545865680492692,
      "learning_rate": 3.6799999999999996e-07,
      "logits/policy_chosen": -2.7706384658813477,
      "logits/policy_rejected": -2.5063834190368652,
      "logits/reference_chosen": -2.7504565715789795,
      "logits/reference_rejected": -1.929320216178894,
      "logps/policy_chosen": -126.52993774414062,
      "logps/policy_rejected": -134.23519897460938,
      "logps/reference_chosen": -138.69195556640625,
      "logps/reference_rejected": -138.32545471191406,
      "loss": 0.6536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12162025272846222,
      "rewards/margins": 0.08071767538785934,
      "rewards/rejected": 0.040902577340602875,
      "step": 92,
      "tokens/chosen_response_len_mean": 315.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 201.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.284769139553738,
      "learning_rate": 3.72e-07,
      "logits/policy_chosen": -2.872082233428955,
      "logits/policy_rejected": -2.6889710426330566,
      "logits/reference_chosen": -2.8072874546051025,
      "logits/reference_rejected": -2.4844841957092285,
      "logps/policy_chosen": -74.13471984863281,
      "logps/policy_rejected": -116.40203857421875,
      "logps/reference_chosen": -80.3674545288086,
      "logps/reference_rejected": -114.1785888671875,
      "loss": 0.6602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06232735142111778,
      "rewards/margins": 0.08456189930438995,
      "rewards/rejected": -0.022234544157981873,
      "step": 93,
      "tokens/chosen_response_len_mean": 162.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 95.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 1.898688154588326,
      "learning_rate": 3.76e-07,
      "logits/policy_chosen": -2.469783306121826,
      "logits/policy_rejected": -2.3272128105163574,
      "logits/reference_chosen": -2.425753593444824,
      "logits/reference_rejected": -2.3546535968780518,
      "logps/policy_chosen": -158.62905883789062,
      "logps/policy_rejected": -291.726318359375,
      "logps/reference_chosen": -168.30911254882812,
      "logps/reference_rejected": -291.2076110839844,
      "loss": 0.6805,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09680052101612091,
      "rewards/margins": 0.10198744386434555,
      "rewards/rejected": -0.005186920985579491,
      "step": 94,
      "tokens/chosen_response_len_mean": 359.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 302.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.9949420570538363,
      "learning_rate": 3.7999999999999996e-07,
      "logits/policy_chosen": -2.8909621238708496,
      "logits/policy_rejected": -2.906135082244873,
      "logits/reference_chosen": -2.792299747467041,
      "logits/reference_rejected": -2.848114252090454,
      "logps/policy_chosen": -50.312339782714844,
      "logps/policy_rejected": -105.15270233154297,
      "logps/reference_chosen": -52.856605529785156,
      "logps/reference_rejected": -108.09819030761719,
      "loss": 0.6483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02544267475605011,
      "rewards/margins": -0.004012217745184898,
      "rewards/rejected": 0.029454896226525307,
      "step": 95,
      "tokens/chosen_response_len_mean": 116.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 120.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.4241955868213405,
      "learning_rate": 3.84e-07,
      "logits/policy_chosen": -2.789196014404297,
      "logits/policy_rejected": -2.543649911880493,
      "logits/reference_chosen": -2.7888166904449463,
      "logits/reference_rejected": -2.5362277030944824,
      "logps/policy_chosen": -127.26165771484375,
      "logps/policy_rejected": -104.05661010742188,
      "logps/reference_chosen": -139.64187622070312,
      "logps/reference_rejected": -108.60116577148438,
      "loss": 0.6656,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12380214780569077,
      "rewards/margins": 0.07835659384727478,
      "rewards/rejected": 0.04544556140899658,
      "step": 96,
      "tokens/chosen_response_len_mean": 381.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 272.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 3.4704291601036683,
      "learning_rate": 3.88e-07,
      "logits/policy_chosen": -2.9072158336639404,
      "logits/policy_rejected": -2.845478057861328,
      "logits/reference_chosen": -2.882009983062744,
      "logits/reference_rejected": -2.943215847015381,
      "logps/policy_chosen": -199.15065002441406,
      "logps/policy_rejected": -345.61346435546875,
      "logps/reference_chosen": -208.939697265625,
      "logps/reference_rejected": -351.9114990234375,
      "loss": 0.6532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09789040684700012,
      "rewards/margins": 0.0349101684987545,
      "rewards/rejected": 0.06298023462295532,
      "step": 97,
      "tokens/chosen_response_len_mean": 443.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 334.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.413585232959013,
      "learning_rate": 3.92e-07,
      "logits/policy_chosen": -2.581110954284668,
      "logits/policy_rejected": -2.5821545124053955,
      "logits/reference_chosen": -2.6214118003845215,
      "logits/reference_rejected": -2.6232800483703613,
      "logps/policy_chosen": -66.4002914428711,
      "logps/policy_rejected": -92.362060546875,
      "logps/reference_chosen": -73.42770385742188,
      "logps/reference_rejected": -96.90080261230469,
      "loss": 0.6682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07027411460876465,
      "rewards/margins": 0.02488674223423004,
      "rewards/rejected": 0.0453873835504055,
      "step": 98,
      "tokens/chosen_response_len_mean": 369.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 353.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.566883896226423,
      "learning_rate": 3.96e-07,
      "logits/policy_chosen": -3.13887357711792,
      "logits/policy_rejected": -3.1172709465026855,
      "logits/reference_chosen": -3.127457618713379,
      "logits/reference_rejected": -3.1038241386413574,
      "logps/policy_chosen": -131.76913452148438,
      "logps/policy_rejected": -274.8529052734375,
      "logps/reference_chosen": -142.9169158935547,
      "logps/reference_rejected": -277.9893798828125,
      "loss": 0.68,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11147774755954742,
      "rewards/margins": 0.08011295646429062,
      "rewards/rejected": 0.031364794820547104,
      "step": 99,
      "tokens/chosen_response_len_mean": 291.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 296.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.7837569692907294,
      "learning_rate": 4e-07,
      "logits/policy_chosen": -3.046888589859009,
      "logits/policy_rejected": -2.9924631118774414,
      "logits/reference_chosen": -3.0203585624694824,
      "logits/reference_rejected": -2.9802844524383545,
      "logps/policy_chosen": -96.66795349121094,
      "logps/policy_rejected": -151.02667236328125,
      "logps/reference_chosen": -106.40546417236328,
      "logps/reference_rejected": -156.30604553222656,
      "loss": 0.6494,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09737517684698105,
      "rewards/margins": 0.04458136856555939,
      "rewards/rejected": 0.05279380828142166,
      "step": 100,
      "tokens/chosen_response_len_mean": 246.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 213.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.5533090774871385,
      "learning_rate": 4.04e-07,
      "logits/policy_chosen": -3.0745935440063477,
      "logits/policy_rejected": -2.9924702644348145,
      "logits/reference_chosen": -3.0464026927948,
      "logits/reference_rejected": -2.9301576614379883,
      "logps/policy_chosen": -130.01454162597656,
      "logps/policy_rejected": -273.28997802734375,
      "logps/reference_chosen": -138.4945831298828,
      "logps/reference_rejected": -277.6722106933594,
      "loss": 0.6659,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08480045944452286,
      "rewards/margins": 0.04097801819443703,
      "rewards/rejected": 0.04382243752479553,
      "step": 101,
      "tokens/chosen_response_len_mean": 351.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 294.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.658053156359227,
      "learning_rate": 4.0799999999999995e-07,
      "logits/policy_chosen": -2.9965174198150635,
      "logits/policy_rejected": -2.756721019744873,
      "logits/reference_chosen": -3.0305070877075195,
      "logits/reference_rejected": -2.5150346755981445,
      "logps/policy_chosen": -150.61041259765625,
      "logps/policy_rejected": -145.7916717529297,
      "logps/reference_chosen": -169.15542602539062,
      "logps/reference_rejected": -147.83485412597656,
      "loss": 0.6388,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18545001745224,
      "rewards/margins": 0.16501817107200623,
      "rewards/rejected": 0.02043186128139496,
      "step": 102,
      "tokens/chosen_response_len_mean": 276.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 211.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 1.8527975992911931,
      "learning_rate": 4.12e-07,
      "logits/policy_chosen": -2.4530792236328125,
      "logits/policy_rejected": -2.4156646728515625,
      "logits/reference_chosen": -2.4479713439941406,
      "logits/reference_rejected": -2.3904917240142822,
      "logps/policy_chosen": -173.00030517578125,
      "logps/policy_rejected": -199.19415283203125,
      "logps/reference_chosen": -189.02093505859375,
      "logps/reference_rejected": -209.84201049804688,
      "loss": 0.6666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16020633280277252,
      "rewards/margins": 0.05372770130634308,
      "rewards/rejected": 0.10647863149642944,
      "step": 103,
      "tokens/chosen_response_len_mean": 309.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.5555937206717187,
      "learning_rate": 4.1599999999999997e-07,
      "logits/policy_chosen": -3.0911600589752197,
      "logits/policy_rejected": -3.0214200019836426,
      "logits/reference_chosen": -3.089872360229492,
      "logits/reference_rejected": -2.882385730743408,
      "logps/policy_chosen": -153.61936950683594,
      "logps/policy_rejected": -227.18930053710938,
      "logps/reference_chosen": -176.565673828125,
      "logps/reference_rejected": -239.63970947265625,
      "loss": 0.665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22946298122406006,
      "rewards/margins": 0.10495913028717041,
      "rewards/rejected": 0.12450385838747025,
      "step": 104,
      "tokens/chosen_response_len_mean": 368.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 301.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.0081558824498096,
      "learning_rate": 4.1999999999999995e-07,
      "logits/policy_chosen": -2.958939790725708,
      "logits/policy_rejected": -2.80196213722229,
      "logits/reference_chosen": -2.944668769836426,
      "logits/reference_rejected": -2.6392948627471924,
      "logps/policy_chosen": -37.521568298339844,
      "logps/policy_rejected": -74.32329559326172,
      "logps/reference_chosen": -41.80109786987305,
      "logps/reference_rejected": -72.32585906982422,
      "loss": 0.664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0427953377366066,
      "rewards/margins": 0.06276971846818924,
      "rewards/rejected": -0.01997438445687294,
      "step": 105,
      "tokens/chosen_response_len_mean": 105.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 85.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.7526269849819696,
      "learning_rate": 4.24e-07,
      "logits/policy_chosen": -3.100095272064209,
      "logits/policy_rejected": -2.8839964866638184,
      "logits/reference_chosen": -3.0812511444091797,
      "logits/reference_rejected": -2.936655044555664,
      "logps/policy_chosen": -207.67698669433594,
      "logps/policy_rejected": -240.35739135742188,
      "logps/reference_chosen": -223.25807189941406,
      "logps/reference_rejected": -239.6046142578125,
      "loss": 0.6494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15581092238426208,
      "rewards/margins": 0.16333873569965363,
      "rewards/rejected": -0.007527794688940048,
      "step": 106,
      "tokens/chosen_response_len_mean": 392.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 203.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 1.9552389682760483,
      "learning_rate": 4.2799999999999997e-07,
      "logits/policy_chosen": -2.922211170196533,
      "logits/policy_rejected": -2.94032621383667,
      "logits/reference_chosen": -2.9038805961608887,
      "logits/reference_rejected": -2.9326558113098145,
      "logps/policy_chosen": -142.2540283203125,
      "logps/policy_rejected": -238.67227172851562,
      "logps/reference_chosen": -151.64273071289062,
      "logps/reference_rejected": -239.12753295898438,
      "loss": 0.6753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09388697892427444,
      "rewards/margins": 0.08933433890342712,
      "rewards/rejected": 0.004552632570266724,
      "step": 107,
      "tokens/chosen_response_len_mean": 347.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 352.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.014699368900832,
      "learning_rate": 4.3199999999999995e-07,
      "logits/policy_chosen": -3.1775548458099365,
      "logits/policy_rejected": -3.03210186958313,
      "logits/reference_chosen": -3.1847496032714844,
      "logits/reference_rejected": -2.531522750854492,
      "logps/policy_chosen": -143.90025329589844,
      "logps/policy_rejected": -291.92388916015625,
      "logps/reference_chosen": -160.6278839111328,
      "logps/reference_rejected": -297.986572265625,
      "loss": 0.6498,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.16727623343467712,
      "rewards/margins": 0.10664979368448257,
      "rewards/rejected": 0.060626450926065445,
      "step": 108,
      "tokens/chosen_response_len_mean": 411.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 381.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.3247511691725236,
      "learning_rate": 4.36e-07,
      "logits/policy_chosen": -3.0070884227752686,
      "logits/policy_rejected": -3.006807565689087,
      "logits/reference_chosen": -2.9760210514068604,
      "logits/reference_rejected": -2.982029438018799,
      "logps/policy_chosen": -117.77191925048828,
      "logps/policy_rejected": -288.87689208984375,
      "logps/reference_chosen": -130.60389709472656,
      "logps/reference_rejected": -292.3191833496094,
      "loss": 0.6517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12831971049308777,
      "rewards/margins": 0.09389686584472656,
      "rewards/rejected": 0.034422844648361206,
      "step": 109,
      "tokens/chosen_response_len_mean": 318.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 302.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.158975765008515,
      "learning_rate": 4.3999999999999997e-07,
      "logits/policy_chosen": -2.50785756111145,
      "logits/policy_rejected": -2.504612684249878,
      "logits/reference_chosen": -2.5301709175109863,
      "logits/reference_rejected": -2.5272464752197266,
      "logps/policy_chosen": -136.28662109375,
      "logps/policy_rejected": -200.89352416992188,
      "logps/reference_chosen": -158.42433166503906,
      "logps/reference_rejected": -212.5179443359375,
      "loss": 0.638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22137700021266937,
      "rewards/margins": 0.10513283312320709,
      "rewards/rejected": 0.11624416708946228,
      "step": 110,
      "tokens/chosen_response_len_mean": 608.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 525.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.8278528577842095,
      "learning_rate": 4.44e-07,
      "logits/policy_chosen": -2.725069999694824,
      "logits/policy_rejected": -2.732088565826416,
      "logits/reference_chosen": -2.6485838890075684,
      "logits/reference_rejected": -2.76961612701416,
      "logps/policy_chosen": -178.09320068359375,
      "logps/policy_rejected": -237.05970764160156,
      "logps/reference_chosen": -192.22482299804688,
      "logps/reference_rejected": -248.2169647216797,
      "loss": 0.6253,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.14131632447242737,
      "rewards/margins": 0.029743948951363564,
      "rewards/rejected": 0.11157237738370895,
      "step": 111,
      "tokens/chosen_response_len_mean": 589.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 556.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.4191485129114714,
      "learning_rate": 4.48e-07,
      "logits/policy_chosen": -2.7914977073669434,
      "logits/policy_rejected": -2.804826259613037,
      "logits/reference_chosen": -2.7042627334594727,
      "logits/reference_rejected": -2.640076160430908,
      "logps/policy_chosen": -146.0531768798828,
      "logps/policy_rejected": -232.5332794189453,
      "logps/reference_chosen": -168.23060607910156,
      "logps/reference_rejected": -237.49801635742188,
      "loss": 0.6295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22177428007125854,
      "rewards/margins": 0.172126904129982,
      "rewards/rejected": 0.049647387117147446,
      "step": 112,
      "tokens/chosen_response_len_mean": 438.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 396.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.331597318165628,
      "learning_rate": 4.5199999999999997e-07,
      "logits/policy_chosen": -3.1060757637023926,
      "logits/policy_rejected": -2.961545467376709,
      "logits/reference_chosen": -3.1052682399749756,
      "logits/reference_rejected": -2.833801746368408,
      "logps/policy_chosen": -127.66785430908203,
      "logps/policy_rejected": -196.163330078125,
      "logps/reference_chosen": -137.27505493164062,
      "logps/reference_rejected": -198.14578247070312,
      "loss": 0.6605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09607198089361191,
      "rewards/margins": 0.07624746859073639,
      "rewards/rejected": 0.01982451230287552,
      "step": 113,
      "tokens/chosen_response_len_mean": 314.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 291.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.750538166446364,
      "learning_rate": 4.56e-07,
      "logits/policy_chosen": -2.7500100135803223,
      "logits/policy_rejected": -2.681910514831543,
      "logits/reference_chosen": -2.789179563522339,
      "logits/reference_rejected": -2.3359477519989014,
      "logps/policy_chosen": -85.98888397216797,
      "logps/policy_rejected": -100.74612426757812,
      "logps/reference_chosen": -98.14977264404297,
      "logps/reference_rejected": -102.4841537475586,
      "loss": 0.6408,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12160897254943848,
      "rewards/margins": 0.10422859340906143,
      "rewards/rejected": 0.017380371689796448,
      "step": 114,
      "tokens/chosen_response_len_mean": 256.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 223.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.5440555248438996,
      "learning_rate": 4.6e-07,
      "logits/policy_chosen": -2.971923351287842,
      "logits/policy_rejected": -2.989212989807129,
      "logits/reference_chosen": -2.9268057346343994,
      "logits/reference_rejected": -2.919334650039673,
      "logps/policy_chosen": -98.5408706665039,
      "logps/policy_rejected": -166.144775390625,
      "logps/reference_chosen": -109.1729965209961,
      "logps/reference_rejected": -167.0085906982422,
      "loss": 0.6505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10632127523422241,
      "rewards/margins": 0.09768319875001907,
      "rewards/rejected": 0.008638077415525913,
      "step": 115,
      "tokens/chosen_response_len_mean": 229.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 254.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.4244019154261474,
      "learning_rate": 4.64e-07,
      "logits/policy_chosen": -2.8886287212371826,
      "logits/policy_rejected": -2.922623872756958,
      "logits/reference_chosen": -2.8570451736450195,
      "logits/reference_rejected": -2.888643741607666,
      "logps/policy_chosen": -113.93293762207031,
      "logps/policy_rejected": -225.3817596435547,
      "logps/reference_chosen": -120.10133361816406,
      "logps/reference_rejected": -223.7104034423828,
      "loss": 0.6579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06168397143483162,
      "rewards/margins": 0.0783977136015892,
      "rewards/rejected": -0.016713742166757584,
      "step": 116,
      "tokens/chosen_response_len_mean": 330.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 318.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.400248188855275,
      "learning_rate": 4.68e-07,
      "logits/policy_chosen": -2.8977608680725098,
      "logits/policy_rejected": -2.9003448486328125,
      "logits/reference_chosen": -2.8988900184631348,
      "logits/reference_rejected": -2.9036717414855957,
      "logps/policy_chosen": -74.31845092773438,
      "logps/policy_rejected": -92.14234924316406,
      "logps/reference_chosen": -78.81231689453125,
      "logps/reference_rejected": -93.07196807861328,
      "loss": 0.6588,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04493863508105278,
      "rewards/margins": 0.035642486065626144,
      "rewards/rejected": 0.00929614994674921,
      "step": 117,
      "tokens/chosen_response_len_mean": 275.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 274.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.4323431386681595,
      "learning_rate": 4.7199999999999994e-07,
      "logits/policy_chosen": -2.6670236587524414,
      "logits/policy_rejected": -2.518751621246338,
      "logits/reference_chosen": -2.666407585144043,
      "logits/reference_rejected": -2.5256845951080322,
      "logps/policy_chosen": -92.44061279296875,
      "logps/policy_rejected": -151.3938446044922,
      "logps/reference_chosen": -100.54401397705078,
      "logps/reference_rejected": -153.23507690429688,
      "loss": 0.6474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08103407174348831,
      "rewards/margins": 0.06262162327766418,
      "rewards/rejected": 0.01841244101524353,
      "step": 118,
      "tokens/chosen_response_len_mean": 302.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 332.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.9673553403256707,
      "learning_rate": 4.76e-07,
      "logits/policy_chosen": -2.9274580478668213,
      "logits/policy_rejected": -2.8594419956207275,
      "logits/reference_chosen": -2.9505772590637207,
      "logits/reference_rejected": -2.850255012512207,
      "logps/policy_chosen": -169.86106872558594,
      "logps/policy_rejected": -316.6098327636719,
      "logps/reference_chosen": -191.77130126953125,
      "logps/reference_rejected": -317.0496520996094,
      "loss": 0.6273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21910235285758972,
      "rewards/margins": 0.21470384299755096,
      "rewards/rejected": 0.004398513585329056,
      "step": 119,
      "tokens/chosen_response_len_mean": 431.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 374.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.3505092556042304,
      "learning_rate": 4.8e-07,
      "logits/policy_chosen": -2.814655303955078,
      "logits/policy_rejected": -2.5319275856018066,
      "logits/reference_chosen": -2.8454208374023438,
      "logits/reference_rejected": -2.315487861633301,
      "logps/policy_chosen": -50.07554626464844,
      "logps/policy_rejected": -110.84510040283203,
      "logps/reference_chosen": -60.05486297607422,
      "logps/reference_rejected": -106.57113647460938,
      "loss": 0.6368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09979310631752014,
      "rewards/margins": 0.14253273606300354,
      "rewards/rejected": -0.0427396334707737,
      "step": 120,
      "tokens/chosen_response_len_mean": 267.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 187.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.8274965378312586,
      "learning_rate": 4.839999999999999e-07,
      "logits/policy_chosen": -2.942972421646118,
      "logits/policy_rejected": -2.8031089305877686,
      "logits/reference_chosen": -2.961139678955078,
      "logits/reference_rejected": -2.5472524166107178,
      "logps/policy_chosen": -130.38365173339844,
      "logps/policy_rejected": -191.11968994140625,
      "logps/reference_chosen": -159.17897033691406,
      "logps/reference_rejected": -199.572021484375,
      "loss": 0.6282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2879532277584076,
      "rewards/margins": 0.2034299373626709,
      "rewards/rejected": 0.0845232680439949,
      "step": 121,
      "tokens/chosen_response_len_mean": 420.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 307.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.886588252399639,
      "learning_rate": 4.879999999999999e-07,
      "logits/policy_chosen": -2.853999376296997,
      "logits/policy_rejected": -2.480708122253418,
      "logits/reference_chosen": -2.710318088531494,
      "logits/reference_rejected": -1.9466197490692139,
      "logps/policy_chosen": -167.15240478515625,
      "logps/policy_rejected": -201.3469696044922,
      "logps/reference_chosen": -181.32119750976562,
      "logps/reference_rejected": -195.66384887695312,
      "loss": 0.6126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14168795943260193,
      "rewards/margins": 0.1985190212726593,
      "rewards/rejected": -0.056831058114767075,
      "step": 122,
      "tokens/chosen_response_len_mean": 326.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 201.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.826829476556088,
      "learning_rate": 4.92e-07,
      "logits/policy_chosen": -2.465135335922241,
      "logits/policy_rejected": -2.4350569248199463,
      "logits/reference_chosen": -2.4818878173828125,
      "logits/reference_rejected": -2.3954761028289795,
      "logps/policy_chosen": -100.21925354003906,
      "logps/policy_rejected": -125.87073516845703,
      "logps/reference_chosen": -126.35533905029297,
      "logps/reference_rejected": -144.03236389160156,
      "loss": 0.6328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.26136085391044617,
      "rewards/margins": 0.07974465936422348,
      "rewards/rejected": 0.1816161870956421,
      "step": 123,
      "tokens/chosen_response_len_mean": 426.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 408.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.5289041457286774,
      "learning_rate": 4.96e-07,
      "logits/policy_chosen": -2.6869404315948486,
      "logits/policy_rejected": -2.773239850997925,
      "logits/reference_chosen": -2.653362512588501,
      "logits/reference_rejected": -2.5821008682250977,
      "logps/policy_chosen": -71.6850357055664,
      "logps/policy_rejected": -108.95183563232422,
      "logps/reference_chosen": -77.52404022216797,
      "logps/reference_rejected": -107.73311614990234,
      "loss": 0.6376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05839002877473831,
      "rewards/margins": 0.07057718932628632,
      "rewards/rejected": -0.012187155894935131,
      "step": 124,
      "tokens/chosen_response_len_mean": 229.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 211.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 3.7452560936185804,
      "learning_rate": 5e-07,
      "logits/policy_chosen": -2.8474879264831543,
      "logits/policy_rejected": -2.77992844581604,
      "logits/reference_chosen": -2.8415565490722656,
      "logits/reference_rejected": -2.658388376235962,
      "logps/policy_chosen": -85.31348419189453,
      "logps/policy_rejected": -100.18048095703125,
      "logps/reference_chosen": -91.71619415283203,
      "logps/reference_rejected": -100.39627838134766,
      "loss": 0.6215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0640270784497261,
      "rewards/margins": 0.06186922267079353,
      "rewards/rejected": 0.00215785950422287,
      "step": 125,
      "tokens/chosen_response_len_mean": 91.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 71.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.791877501388191,
      "learning_rate": 4.999990147435196e-07,
      "logits/policy_chosen": -2.772634506225586,
      "logits/policy_rejected": -2.7663345336914062,
      "logits/reference_chosen": -2.8182811737060547,
      "logits/reference_rejected": -2.821171760559082,
      "logps/policy_chosen": -114.60906982421875,
      "logps/policy_rejected": -122.4944839477539,
      "logps/reference_chosen": -141.17056274414062,
      "logps/reference_rejected": -135.64431762695312,
      "loss": 0.63,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2656148672103882,
      "rewards/margins": 0.13411659002304077,
      "rewards/rejected": 0.1314982771873474,
      "step": 126,
      "tokens/chosen_response_len_mean": 458.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 332.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.692910890812567,
      "learning_rate": 4.99996058981844e-07,
      "logits/policy_chosen": -2.5161116123199463,
      "logits/policy_rejected": -2.3703317642211914,
      "logits/reference_chosen": -2.4154701232910156,
      "logits/reference_rejected": -2.4023566246032715,
      "logps/policy_chosen": -72.83512115478516,
      "logps/policy_rejected": -136.28607177734375,
      "logps/reference_chosen": -82.63104248046875,
      "logps/reference_rejected": -140.04986572265625,
      "loss": 0.6369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09795919060707092,
      "rewards/margins": 0.06032119691371918,
      "rewards/rejected": 0.037637993693351746,
      "step": 127,
      "tokens/chosen_response_len_mean": 416.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 410.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.859419025019686,
      "learning_rate": 4.999911327382711e-07,
      "logits/policy_chosen": -2.0673351287841797,
      "logits/policy_rejected": -1.9885450601577759,
      "logits/reference_chosen": -2.1499083042144775,
      "logits/reference_rejected": -1.6471858024597168,
      "logps/policy_chosen": -57.06989669799805,
      "logps/policy_rejected": -87.03865814208984,
      "logps/reference_chosen": -69.13873291015625,
      "logps/reference_rejected": -83.46507263183594,
      "loss": 0.5998,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12068840861320496,
      "rewards/margins": 0.15642426908016205,
      "rewards/rejected": -0.035735856741666794,
      "step": 128,
      "tokens/chosen_response_len_mean": 343.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.8759886658807288,
      "learning_rate": 4.999842360516294e-07,
      "logits/policy_chosen": -3.1045093536376953,
      "logits/policy_rejected": -3.1223738193511963,
      "logits/reference_chosen": -3.09148907661438,
      "logits/reference_rejected": -3.091776132583618,
      "logps/policy_chosen": -40.253719329833984,
      "logps/policy_rejected": -93.64241027832031,
      "logps/reference_chosen": -43.66665267944336,
      "logps/reference_rejected": -92.32077026367188,
      "loss": 0.5883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03412933275103569,
      "rewards/margins": 0.047345712780952454,
      "rewards/rejected": -0.01321637723594904,
      "step": 129,
      "tokens/chosen_response_len_mean": 134.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 142.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.396608033911008,
      "learning_rate": 4.999753689762792e-07,
      "logits/policy_chosen": -2.7669222354888916,
      "logits/policy_rejected": -3.0244264602661133,
      "logits/reference_chosen": -2.7182681560516357,
      "logits/reference_rejected": -3.016852855682373,
      "logps/policy_chosen": -49.25522994995117,
      "logps/policy_rejected": -209.12026977539062,
      "logps/reference_chosen": -58.14155197143555,
      "logps/reference_rejected": -213.75338745117188,
      "loss": 0.6532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08886321634054184,
      "rewards/margins": 0.04253204166889191,
      "rewards/rejected": 0.04633117839694023,
      "step": 130,
      "tokens/chosen_response_len_mean": 140.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 397.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.8338085411833127,
      "learning_rate": 4.999645315821111e-07,
      "logits/policy_chosen": -2.177436590194702,
      "logits/policy_rejected": -2.1923160552978516,
      "logits/reference_chosen": -2.23976993560791,
      "logits/reference_rejected": -2.227308511734009,
      "logps/policy_chosen": -92.42753601074219,
      "logps/policy_rejected": -104.93522644042969,
      "logps/reference_chosen": -111.59609985351562,
      "logps/reference_rejected": -120.66436004638672,
      "loss": 0.6227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19168560206890106,
      "rewards/margins": 0.03439423814415932,
      "rewards/rejected": 0.15729136765003204,
      "step": 131,
      "tokens/chosen_response_len_mean": 428.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 393.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.537615272647266,
      "learning_rate": 4.99951723954546e-07,
      "logits/policy_chosen": -2.6528267860412598,
      "logits/policy_rejected": -2.7896065711975098,
      "logits/reference_chosen": -2.3186991214752197,
      "logits/reference_rejected": -2.7976412773132324,
      "logps/policy_chosen": -35.68031311035156,
      "logps/policy_rejected": -72.86119079589844,
      "logps/reference_chosen": -42.128787994384766,
      "logps/reference_rejected": -76.47801208496094,
      "loss": 0.6365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06448473036289215,
      "rewards/margins": 0.028316427022218704,
      "rewards/rejected": 0.03616830334067345,
      "step": 132,
      "tokens/chosen_response_len_mean": 226.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 273.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.952022770646515,
      "learning_rate": 4.999369461945343e-07,
      "logits/policy_chosen": -3.1463866233825684,
      "logits/policy_rejected": -2.8932509422302246,
      "logits/reference_chosen": -3.141604423522949,
      "logits/reference_rejected": -2.444639205932617,
      "logps/policy_chosen": -83.71105194091797,
      "logps/policy_rejected": -113.13372802734375,
      "logps/reference_chosen": -94.0230941772461,
      "logps/reference_rejected": -108.04383850097656,
      "loss": 0.5892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10312040150165558,
      "rewards/margins": 0.15401925146579742,
      "rewards/rejected": -0.05089886486530304,
      "step": 133,
      "tokens/chosen_response_len_mean": 249.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 178.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 3.540843119893174,
      "learning_rate": 4.999201984185552e-07,
      "logits/policy_chosen": -3.0958125591278076,
      "logits/policy_rejected": -3.042353868484497,
      "logits/reference_chosen": -3.133866310119629,
      "logits/reference_rejected": -3.0950450897216797,
      "logps/policy_chosen": -236.39605712890625,
      "logps/policy_rejected": -330.2846374511719,
      "logps/reference_chosen": -259.0321350097656,
      "logps/reference_rejected": -336.391357421875,
      "loss": 0.5663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22636055946350098,
      "rewards/margins": 0.16529357433319092,
      "rewards/rejected": 0.06106698513031006,
      "step": 134,
      "tokens/chosen_response_len_mean": 717.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 639.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.981444512825392,
      "learning_rate": 4.999014807586154e-07,
      "logits/policy_chosen": -2.3856396675109863,
      "logits/policy_rejected": -2.1970813274383545,
      "logits/reference_chosen": -2.4693095684051514,
      "logits/reference_rejected": -1.5482537746429443,
      "logps/policy_chosen": -142.9355926513672,
      "logps/policy_rejected": -189.60543823242188,
      "logps/reference_chosen": -164.72271728515625,
      "logps/reference_rejected": -189.248046875,
      "loss": 0.6065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2178712785243988,
      "rewards/margins": 0.22144514322280884,
      "rewards/rejected": -0.003573855385184288,
      "step": 135,
      "tokens/chosen_response_len_mean": 339.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 276.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.4024504350993654,
      "learning_rate": 4.998807933622484e-07,
      "logits/policy_chosen": -3.1550679206848145,
      "logits/policy_rejected": -3.1557650566101074,
      "logits/reference_chosen": -3.1761221885681152,
      "logits/reference_rejected": -3.135535955429077,
      "logps/policy_chosen": -118.20024871826172,
      "logps/policy_rejected": -264.43560791015625,
      "logps/reference_chosen": -130.25527954101562,
      "logps/reference_rejected": -264.16741943359375,
      "loss": 0.6336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12055033445358276,
      "rewards/margins": 0.12323223054409027,
      "rewards/rejected": -0.002681899815797806,
      "step": 136,
      "tokens/chosen_response_len_mean": 354.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 386.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.8382305196380684,
      "learning_rate": 4.998581363925135e-07,
      "logits/policy_chosen": -2.2047266960144043,
      "logits/policy_rejected": -2.433465003967285,
      "logits/reference_chosen": -2.275531530380249,
      "logits/reference_rejected": -2.245540142059326,
      "logps/policy_chosen": -378.1270751953125,
      "logps/policy_rejected": -344.7900085449219,
      "logps/reference_chosen": -416.6529541015625,
      "logps/reference_rejected": -347.785888671875,
      "loss": 0.5804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3852587938308716,
      "rewards/margins": 0.3552997410297394,
      "rewards/rejected": 0.02995903044939041,
      "step": 137,
      "tokens/chosen_response_len_mean": 584.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 427.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.1385240477591636,
      "learning_rate": 4.998335100279941e-07,
      "logits/policy_chosen": -2.6462714672088623,
      "logits/policy_rejected": -2.658045768737793,
      "logits/reference_chosen": -2.729498863220215,
      "logits/reference_rejected": -2.694502830505371,
      "logps/policy_chosen": -93.70889282226562,
      "logps/policy_rejected": -182.60643005371094,
      "logps/reference_chosen": -114.53910827636719,
      "logps/reference_rejected": -185.69065856933594,
      "loss": 0.6218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2083020955324173,
      "rewards/margins": 0.17745977640151978,
      "rewards/rejected": 0.030842319130897522,
      "step": 138,
      "tokens/chosen_response_len_mean": 282.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 247.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.116685308038921,
      "learning_rate": 4.998069144627964e-07,
      "logits/policy_chosen": -2.870461940765381,
      "logits/policy_rejected": -2.6627469062805176,
      "logits/reference_chosen": -2.924255132675171,
      "logits/reference_rejected": -2.6669623851776123,
      "logps/policy_chosen": -149.49957275390625,
      "logps/policy_rejected": -133.25009155273438,
      "logps/reference_chosen": -191.32994079589844,
      "logps/reference_rejected": -157.3734588623047,
      "loss": 0.6358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4183034896850586,
      "rewards/margins": 0.1770700067281723,
      "rewards/rejected": 0.2412334829568863,
      "step": 139,
      "tokens/chosen_response_len_mean": 457.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 308.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.5870338720368387,
      "learning_rate": 4.99778349906548e-07,
      "logits/policy_chosen": -2.9523699283599854,
      "logits/policy_rejected": -2.9597270488739014,
      "logits/reference_chosen": -2.9815268516540527,
      "logits/reference_rejected": -2.9721755981445312,
      "logps/policy_chosen": -45.849361419677734,
      "logps/policy_rejected": -106.37723541259766,
      "logps/reference_chosen": -56.053375244140625,
      "logps/reference_rejected": -102.900390625,
      "loss": 0.6334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10204015672206879,
      "rewards/margins": 0.13680851459503174,
      "rewards/rejected": -0.03476836904883385,
      "step": 140,
      "tokens/chosen_response_len_mean": 174.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 184.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 3.1171856643556626,
      "learning_rate": 4.997478165843964e-07,
      "logits/policy_chosen": -2.8171520233154297,
      "logits/policy_rejected": -2.8751494884490967,
      "logits/reference_chosen": -2.9145286083221436,
      "logits/reference_rejected": -2.9242196083068848,
      "logps/policy_chosen": -170.06459045410156,
      "logps/policy_rejected": -296.7114562988281,
      "logps/reference_chosen": -197.37327575683594,
      "logps/reference_rejected": -302.43438720703125,
      "loss": 0.5862,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.27308669686317444,
      "rewards/margins": 0.21585765480995178,
      "rewards/rejected": 0.057229042053222656,
      "step": 141,
      "tokens/chosen_response_len_mean": 416.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 422.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 3.0084655845363835,
      "learning_rate": 4.997153147370065e-07,
      "logits/policy_chosen": -2.85317325592041,
      "logits/policy_rejected": -2.8581879138946533,
      "logits/reference_chosen": -2.7449471950531006,
      "logits/reference_rejected": -2.9342892169952393,
      "logps/policy_chosen": -95.86734008789062,
      "logps/policy_rejected": -255.03668212890625,
      "logps/reference_chosen": -114.49011993408203,
      "logps/reference_rejected": -234.34010314941406,
      "loss": 0.5691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18622779846191406,
      "rewards/margins": 0.39319345355033875,
      "rewards/rejected": -0.20696566998958588,
      "step": 142,
      "tokens/chosen_response_len_mean": 348.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 233.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.8051570403665753,
      "learning_rate": 4.996808446205598e-07,
      "logits/policy_chosen": -2.9738688468933105,
      "logits/policy_rejected": -2.8889176845550537,
      "logits/reference_chosen": -3.072221517562866,
      "logits/reference_rejected": -2.744046449661255,
      "logps/policy_chosen": -106.87481689453125,
      "logps/policy_rejected": -214.15478515625,
      "logps/reference_chosen": -126.41868591308594,
      "logps/reference_rejected": -216.4077606201172,
      "loss": 0.6244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19543880224227905,
      "rewards/margins": 0.17290906608104706,
      "rewards/rejected": 0.022529736161231995,
      "step": 143,
      "tokens/chosen_response_len_mean": 340.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 3.1099351541314864,
      "learning_rate": 4.996444065067515e-07,
      "logits/policy_chosen": -2.849621534347534,
      "logits/policy_rejected": -2.8288486003875732,
      "logits/reference_chosen": -2.9862475395202637,
      "logits/reference_rejected": -2.9479100704193115,
      "logps/policy_chosen": -331.7215576171875,
      "logps/policy_rejected": -571.941650390625,
      "logps/reference_chosen": -382.40533447265625,
      "logps/reference_rejected": -549.6744995117188,
      "loss": 0.5803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5068381428718567,
      "rewards/margins": 0.7295098304748535,
      "rewards/rejected": -0.2226717472076416,
      "step": 144,
      "tokens/chosen_response_len_mean": 687.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 550.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.712783580670268,
      "learning_rate": 4.996060006827887e-07,
      "logits/policy_chosen": -2.882990837097168,
      "logits/policy_rejected": -2.8665995597839355,
      "logits/reference_chosen": -2.9348299503326416,
      "logits/reference_rejected": -2.9280238151550293,
      "logps/policy_chosen": -75.93386840820312,
      "logps/policy_rejected": -159.058837890625,
      "logps/reference_chosen": -91.52436828613281,
      "logps/reference_rejected": -146.97247314453125,
      "loss": 0.5965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15590497851371765,
      "rewards/margins": 0.2767685055732727,
      "rewards/rejected": -0.12086354941129684,
      "step": 145,
      "tokens/chosen_response_len_mean": 248.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 191.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.364187725655967,
      "learning_rate": 4.995656274513881e-07,
      "logits/policy_chosen": -3.0416619777679443,
      "logits/policy_rejected": -2.995178699493408,
      "logits/reference_chosen": -3.053309440612793,
      "logits/reference_rejected": -3.0008325576782227,
      "logps/policy_chosen": -90.75337982177734,
      "logps/policy_rejected": -127.47672271728516,
      "logps/reference_chosen": -102.95506286621094,
      "logps/reference_rejected": -132.29486083984375,
      "loss": 0.6275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1220167949795723,
      "rewards/margins": 0.07383548468351364,
      "rewards/rejected": 0.04818130284547806,
      "step": 146,
      "tokens/chosen_response_len_mean": 403.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 352.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 3.163944044368287,
      "learning_rate": 4.995232871307736e-07,
      "logits/policy_chosen": -3.0712456703186035,
      "logits/policy_rejected": -2.916374921798706,
      "logits/reference_chosen": -3.115262746810913,
      "logits/reference_rejected": -2.6205530166625977,
      "logps/policy_chosen": -104.03935241699219,
      "logps/policy_rejected": -156.13577270507812,
      "logps/reference_chosen": -121.57315063476562,
      "logps/reference_rejected": -153.22605895996094,
      "loss": 0.583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17533791065216064,
      "rewards/margins": 0.2044350802898407,
      "rewards/rejected": -0.029097169637680054,
      "step": 147,
      "tokens/chosen_response_len_mean": 273.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 197.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.8981364214624588,
      "learning_rate": 4.994789800546737e-07,
      "logits/policy_chosen": -2.519289970397949,
      "logits/policy_rejected": -2.3818559646606445,
      "logits/reference_chosen": -2.5594940185546875,
      "logits/reference_rejected": -2.2251200675964355,
      "logps/policy_chosen": -64.7489242553711,
      "logps/policy_rejected": -90.21540832519531,
      "logps/reference_chosen": -80.6169204711914,
      "logps/reference_rejected": -92.92266845703125,
      "loss": 0.5887,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15867996215820312,
      "rewards/margins": 0.13160738348960876,
      "rewards/rejected": 0.027072593569755554,
      "step": 148,
      "tokens/chosen_response_len_mean": 252.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 195.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.7118945452462073,
      "learning_rate": 4.994327065723192e-07,
      "logits/policy_chosen": -2.6516730785369873,
      "logits/policy_rejected": -2.5235743522644043,
      "logits/reference_chosen": -2.629504442214966,
      "logits/reference_rejected": -2.4736945629119873,
      "logps/policy_chosen": -80.69113159179688,
      "logps/policy_rejected": -134.16526794433594,
      "logps/reference_chosen": -103.33537292480469,
      "logps/reference_rejected": -144.43687438964844,
      "loss": 0.5924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2264423370361328,
      "rewards/margins": 0.12372627854347229,
      "rewards/rejected": 0.10271605104207993,
      "step": 149,
      "tokens/chosen_response_len_mean": 244.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 244.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.5381060398035205,
      "learning_rate": 4.9938446704844e-07,
      "logits/policy_chosen": -2.8819587230682373,
      "logits/policy_rejected": -2.9721384048461914,
      "logits/reference_chosen": -2.7422800064086914,
      "logits/reference_rejected": -2.9950976371765137,
      "logps/policy_chosen": -57.52130126953125,
      "logps/policy_rejected": -120.23619079589844,
      "logps/reference_chosen": -59.788352966308594,
      "logps/reference_rejected": -115.03448486328125,
      "loss": 0.619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.022670526057481766,
      "rewards/margins": 0.07468754798173904,
      "rewards/rejected": -0.05201702564954758,
      "step": 150,
      "tokens/chosen_response_len_mean": 145.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 157.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.8034120172472052,
      "learning_rate": 4.993342618632626e-07,
      "logits/policy_chosen": -3.0852291584014893,
      "logits/policy_rejected": -2.92305850982666,
      "logits/reference_chosen": -3.162454128265381,
      "logits/reference_rejected": -2.4117138385772705,
      "logps/policy_chosen": -101.7428207397461,
      "logps/policy_rejected": -242.9844207763672,
      "logps/reference_chosen": -119.85108184814453,
      "logps/reference_rejected": -215.4344940185547,
      "loss": 0.5702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18108254671096802,
      "rewards/margins": 0.45658186078071594,
      "rewards/rejected": -0.2754993140697479,
      "step": 151,
      "tokens/chosen_response_len_mean": 318.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 234.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.9459543515337776,
      "learning_rate": 4.992820914125069e-07,
      "logits/policy_chosen": -2.8646183013916016,
      "logits/policy_rejected": -2.754851818084717,
      "logits/reference_chosen": -3.0292770862579346,
      "logits/reference_rejected": -2.5228896141052246,
      "logps/policy_chosen": -150.56875610351562,
      "logps/policy_rejected": -277.60125732421875,
      "logps/reference_chosen": -179.77772521972656,
      "logps/reference_rejected": -275.3694152832031,
      "loss": 0.586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2920897901058197,
      "rewards/margins": 0.31440818309783936,
      "rewards/rejected": -0.022318381816148758,
      "step": 152,
      "tokens/chosen_response_len_mean": 543.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 491.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.9972427731212954,
      "learning_rate": 4.99227956107383e-07,
      "logits/policy_chosen": -2.133497714996338,
      "logits/policy_rejected": -2.275707721710205,
      "logits/reference_chosen": -2.4100184440612793,
      "logits/reference_rejected": -2.27435040473938,
      "logps/policy_chosen": -149.89268493652344,
      "logps/policy_rejected": -225.88043212890625,
      "logps/reference_chosen": -195.48631286621094,
      "logps/reference_rejected": -232.33531188964844,
      "loss": 0.5712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4559362828731537,
      "rewards/margins": 0.3913875222206116,
      "rewards/rejected": 0.06454875320196152,
      "step": 153,
      "tokens/chosen_response_len_mean": 551.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 501.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.6455429598648053,
      "learning_rate": 4.991718563745882e-07,
      "logits/policy_chosen": -3.022681951522827,
      "logits/policy_rejected": -3.033928155899048,
      "logits/reference_chosen": -3.0507426261901855,
      "logits/reference_rejected": -3.0070152282714844,
      "logps/policy_chosen": -17.34947395324707,
      "logps/policy_rejected": -55.760353088378906,
      "logps/reference_chosen": -20.67547607421875,
      "logps/reference_rejected": -50.72275924682617,
      "loss": 0.5857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03326002135872841,
      "rewards/margins": 0.08363597095012665,
      "rewards/rejected": -0.05037594214081764,
      "step": 154,
      "tokens/chosen_response_len_mean": 28.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 28.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.188503580126624,
      "learning_rate": 4.991137926563035e-07,
      "logits/policy_chosen": -2.8026483058929443,
      "logits/policy_rejected": -2.840829372406006,
      "logits/reference_chosen": -2.866769790649414,
      "logits/reference_rejected": -2.868910789489746,
      "logps/policy_chosen": -45.941871643066406,
      "logps/policy_rejected": -90.47376251220703,
      "logps/reference_chosen": -53.47924041748047,
      "logps/reference_rejected": -84.99751281738281,
      "loss": 0.6102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07537368685007095,
      "rewards/margins": 0.1301361620426178,
      "rewards/rejected": -0.054762475192546844,
      "step": 155,
      "tokens/chosen_response_len_mean": 173.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 133.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.5667127080808974,
      "learning_rate": 4.990537654101903e-07,
      "logits/policy_chosen": -2.980593681335449,
      "logits/policy_rejected": -3.0731048583984375,
      "logits/reference_chosen": -3.066528797149658,
      "logits/reference_rejected": -3.1092565059661865,
      "logps/policy_chosen": -92.3929672241211,
      "logps/policy_rejected": -223.2591552734375,
      "logps/reference_chosen": -117.49671936035156,
      "logps/reference_rejected": -224.93527221679688,
      "loss": 0.5677,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.25103750824928284,
      "rewards/margins": 0.2342762053012848,
      "rewards/rejected": 0.016761288046836853,
      "step": 156,
      "tokens/chosen_response_len_mean": 291.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 321.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.742556033679143,
      "learning_rate": 4.989917751093862e-07,
      "logits/policy_chosen": -2.9195590019226074,
      "logits/policy_rejected": -2.7180957794189453,
      "logits/reference_chosen": -3.0530753135681152,
      "logits/reference_rejected": -2.1471331119537354,
      "logps/policy_chosen": -86.05413818359375,
      "logps/policy_rejected": -171.83856201171875,
      "logps/reference_chosen": -108.33873748779297,
      "logps/reference_rejected": -158.24143981933594,
      "loss": 0.5624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22284595668315887,
      "rewards/margins": 0.35881713032722473,
      "rewards/rejected": -0.13597117364406586,
      "step": 157,
      "tokens/chosen_response_len_mean": 300.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 193.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.320383809339318,
      "learning_rate": 4.989278222425021e-07,
      "logits/policy_chosen": -2.9507193565368652,
      "logits/policy_rejected": -3.0587496757507324,
      "logits/reference_chosen": -3.0194993019104004,
      "logits/reference_rejected": -3.0736992359161377,
      "logps/policy_chosen": -103.41101837158203,
      "logps/policy_rejected": -312.6053466796875,
      "logps/reference_chosen": -126.41258239746094,
      "logps/reference_rejected": -292.84515380859375,
      "loss": 0.5961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2300155907869339,
      "rewards/margins": 0.4276174306869507,
      "rewards/rejected": -0.19760185480117798,
      "step": 158,
      "tokens/chosen_response_len_mean": 338.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 361.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.516259723460725,
      "learning_rate": 4.988619073136179e-07,
      "logits/policy_chosen": -2.578339099884033,
      "logits/policy_rejected": -2.6243929862976074,
      "logits/reference_chosen": -2.651563882827759,
      "logits/reference_rejected": -2.7075228691101074,
      "logps/policy_chosen": -60.65949249267578,
      "logps/policy_rejected": -124.75245666503906,
      "logps/reference_chosen": -77.07334899902344,
      "logps/reference_rejected": -121.59002685546875,
      "loss": 0.5969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16413861513137817,
      "rewards/margins": 0.19576290249824524,
      "rewards/rejected": -0.031624287366867065,
      "step": 159,
      "tokens/chosen_response_len_mean": 344.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 347.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.715123355655463,
      "learning_rate": 4.987940308422783e-07,
      "logits/policy_chosen": -2.0114169120788574,
      "logits/policy_rejected": -2.0045886039733887,
      "logits/reference_chosen": -2.092721700668335,
      "logits/reference_rejected": -1.7612866163253784,
      "logps/policy_chosen": -84.94512176513672,
      "logps/policy_rejected": -133.75323486328125,
      "logps/reference_chosen": -107.3111801147461,
      "logps/reference_rejected": -128.45538330078125,
      "loss": 0.5549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22366058826446533,
      "rewards/margins": 0.27663907408714294,
      "rewards/rejected": -0.05297844856977463,
      "step": 160,
      "tokens/chosen_response_len_mean": 425.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 374.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.6648898364230735,
      "learning_rate": 4.987241933634892e-07,
      "logits/policy_chosen": -2.6987576484680176,
      "logits/policy_rejected": -2.7210419178009033,
      "logits/reference_chosen": -2.804107189178467,
      "logits/reference_rejected": -2.7485485076904297,
      "logps/policy_chosen": -101.68658447265625,
      "logps/policy_rejected": -234.5952606201172,
      "logps/reference_chosen": -123.14704132080078,
      "logps/reference_rejected": -228.36183166503906,
      "loss": 0.5769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21460454165935516,
      "rewards/margins": 0.276938796043396,
      "rewards/rejected": -0.06233424320816994,
      "step": 161,
      "tokens/chosen_response_len_mean": 409.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 407.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.82572590776649,
      "learning_rate": 4.986523954277134e-07,
      "logits/policy_chosen": -2.5539088249206543,
      "logits/policy_rejected": -2.5690407752990723,
      "logits/reference_chosen": -2.634115695953369,
      "logits/reference_rejected": -2.615056037902832,
      "logps/policy_chosen": -37.4378662109375,
      "logps/policy_rejected": -93.33854675292969,
      "logps/reference_chosen": -45.8214225769043,
      "logps/reference_rejected": -89.81768798828125,
      "loss": 0.5852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08383554965257645,
      "rewards/margins": 0.11904413998126984,
      "rewards/rejected": -0.03520859405398369,
      "step": 162,
      "tokens/chosen_response_len_mean": 228.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 240.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.7170854054640863,
      "learning_rate": 4.985786376008658e-07,
      "logits/policy_chosen": -2.820416212081909,
      "logits/policy_rejected": -2.79477858543396,
      "logits/reference_chosen": -2.9494552612304688,
      "logits/reference_rejected": -2.837876081466675,
      "logps/policy_chosen": -71.54603576660156,
      "logps/policy_rejected": -156.01953125,
      "logps/reference_chosen": -90.15393829345703,
      "logps/reference_rejected": -154.65646362304688,
      "loss": 0.5657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1860790103673935,
      "rewards/margins": 0.1997096687555313,
      "rewards/rejected": -0.01363067515194416,
      "step": 163,
      "tokens/chosen_response_len_mean": 302.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 296.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.3740765633517396,
      "learning_rate": 4.985029204643094e-07,
      "logits/policy_chosen": -2.747786045074463,
      "logits/policy_rejected": -2.7663800716400146,
      "logits/reference_chosen": -2.8246467113494873,
      "logits/reference_rejected": -2.6436727046966553,
      "logps/policy_chosen": -58.324737548828125,
      "logps/policy_rejected": -223.62765502929688,
      "logps/reference_chosen": -85.9714126586914,
      "logps/reference_rejected": -203.6089630126953,
      "loss": 0.6018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27646681666374207,
      "rewards/margins": 0.4766535758972168,
      "rewards/rejected": -0.20018675923347473,
      "step": 164,
      "tokens/chosen_response_len_mean": 223.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 216.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.188806416183682,
      "learning_rate": 4.984252446148508e-07,
      "logits/policy_chosen": -2.701019287109375,
      "logits/policy_rejected": -2.729661226272583,
      "logits/reference_chosen": -2.8306493759155273,
      "logits/reference_rejected": -2.8457367420196533,
      "logps/policy_chosen": -154.78518676757812,
      "logps/policy_rejected": -214.3829345703125,
      "logps/reference_chosen": -194.75637817382812,
      "logps/reference_rejected": -246.5928497314453,
      "loss": 0.6206,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.39971184730529785,
      "rewards/margins": 0.07761266827583313,
      "rewards/rejected": 0.3220991790294647,
      "step": 165,
      "tokens/chosen_response_len_mean": 429.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 445.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.8889287147059104,
      "learning_rate": 4.983456106647348e-07,
      "logits/policy_chosen": -2.861962080001831,
      "logits/policy_rejected": -2.8529515266418457,
      "logits/reference_chosen": -3.064021587371826,
      "logits/reference_rejected": -2.927248001098633,
      "logps/policy_chosen": -146.7970428466797,
      "logps/policy_rejected": -283.18389892578125,
      "logps/reference_chosen": -181.90396118164062,
      "logps/reference_rejected": -277.1955871582031,
      "loss": 0.5642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35106930136680603,
      "rewards/margins": 0.4109525978565216,
      "rewards/rejected": -0.05988328531384468,
      "step": 166,
      "tokens/chosen_response_len_mean": 468.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 421.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.9841019270826292,
      "learning_rate": 4.982640192416404e-07,
      "logits/policy_chosen": -2.6893157958984375,
      "logits/policy_rejected": -2.681230068206787,
      "logits/reference_chosen": -2.948234796524048,
      "logits/reference_rejected": -2.248135566711426,
      "logps/policy_chosen": -122.96820831298828,
      "logps/policy_rejected": -197.66404724121094,
      "logps/reference_chosen": -155.029541015625,
      "logps/reference_rejected": -194.3126678466797,
      "loss": 0.5642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32061344385147095,
      "rewards/margins": 0.35412728786468506,
      "rewards/rejected": -0.0335138775408268,
      "step": 167,
      "tokens/chosen_response_len_mean": 452.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 366.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.766687221527344,
      "learning_rate": 4.981804709886755e-07,
      "logits/policy_chosen": -2.4212305545806885,
      "logits/policy_rejected": -2.4506514072418213,
      "logits/reference_chosen": -2.6806509494781494,
      "logits/reference_rejected": -2.608292579650879,
      "logps/policy_chosen": -60.471797943115234,
      "logps/policy_rejected": -120.82171630859375,
      "logps/reference_chosen": -92.31079864501953,
      "logps/reference_rejected": -130.82656860351562,
      "loss": 0.54,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31838998198509216,
      "rewards/margins": 0.21834133565425873,
      "rewards/rejected": 0.10004864633083344,
      "step": 168,
      "tokens/chosen_response_len_mean": 324.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.1828651372724637,
      "learning_rate": 4.980949665643719e-07,
      "logits/policy_chosen": -2.7441906929016113,
      "logits/policy_rejected": -2.7218408584594727,
      "logits/reference_chosen": -2.8971827030181885,
      "logits/reference_rejected": -2.860405445098877,
      "logps/policy_chosen": -92.7531509399414,
      "logps/policy_rejected": -210.39797973632812,
      "logps/reference_chosen": -123.75934600830078,
      "logps/reference_rejected": -200.2626190185547,
      "loss": 0.5755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3100619614124298,
      "rewards/margins": 0.41141554713249207,
      "rewards/rejected": -0.10135360062122345,
      "step": 169,
      "tokens/chosen_response_len_mean": 517.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 465.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.9989447327164425,
      "learning_rate": 4.980075066426795e-07,
      "logits/policy_chosen": -1.6698166131973267,
      "logits/policy_rejected": -1.7434771060943604,
      "logits/reference_chosen": -1.803000569343567,
      "logits/reference_rejected": -1.8658090829849243,
      "logps/policy_chosen": -61.05796813964844,
      "logps/policy_rejected": -89.74996948242188,
      "logps/reference_chosen": -77.35304260253906,
      "logps/reference_rejected": -98.12295532226562,
      "loss": 0.5561,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1629507839679718,
      "rewards/margins": 0.07922087609767914,
      "rewards/rejected": 0.08372990787029266,
      "step": 170,
      "tokens/chosen_response_len_mean": 243.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 245.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.5554431416025243,
      "learning_rate": 4.979180919129624e-07,
      "logits/policy_chosen": -2.85784912109375,
      "logits/policy_rejected": -2.838977098464966,
      "logits/reference_chosen": -2.8757684230804443,
      "logits/reference_rejected": -2.8608899116516113,
      "logps/policy_chosen": -53.759185791015625,
      "logps/policy_rejected": -90.618408203125,
      "logps/reference_chosen": -62.647789001464844,
      "logps/reference_rejected": -86.81078338623047,
      "loss": 0.5702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08888600766658783,
      "rewards/margins": 0.1269623190164566,
      "rewards/rejected": -0.03807631880044937,
      "step": 171,
      "tokens/chosen_response_len_mean": 207.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 209.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.7869612955855776,
      "learning_rate": 4.978267230799918e-07,
      "logits/policy_chosen": -2.0658278465270996,
      "logits/policy_rejected": -2.2496557235717773,
      "logits/reference_chosen": -2.1524839401245117,
      "logits/reference_rejected": -2.3013737201690674,
      "logps/policy_chosen": -86.49990844726562,
      "logps/policy_rejected": -197.741943359375,
      "logps/reference_chosen": -109.14804077148438,
      "logps/reference_rejected": -182.33616638183594,
      "loss": 0.5549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22648128867149353,
      "rewards/margins": 0.380539208650589,
      "rewards/rejected": -0.15405790507793427,
      "step": 172,
      "tokens/chosen_response_len_mean": 443.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 439.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.9395952260087506,
      "learning_rate": 4.977334008639417e-07,
      "logits/policy_chosen": -2.605189800262451,
      "logits/policy_rejected": -2.5856776237487793,
      "logits/reference_chosen": -2.778407335281372,
      "logits/reference_rejected": -2.793548583984375,
      "logps/policy_chosen": -112.27513122558594,
      "logps/policy_rejected": -266.2890625,
      "logps/reference_chosen": -140.7950439453125,
      "logps/reference_rejected": -241.72708129882812,
      "loss": 0.5051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28519904613494873,
      "rewards/margins": 0.5308188796043396,
      "rewards/rejected": -0.24561986327171326,
      "step": 173,
      "tokens/chosen_response_len_mean": 462.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 293.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.7423154795634153,
      "learning_rate": 4.976381260003826e-07,
      "logits/policy_chosen": -2.7179572582244873,
      "logits/policy_rejected": -2.756556510925293,
      "logits/reference_chosen": -2.8246583938598633,
      "logits/reference_rejected": -2.793865203857422,
      "logps/policy_chosen": -80.221923828125,
      "logps/policy_rejected": -204.06829833984375,
      "logps/reference_chosen": -96.3402328491211,
      "logps/reference_rejected": -181.1507568359375,
      "loss": 0.5916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16118302941322327,
      "rewards/margins": 0.3903582692146301,
      "rewards/rejected": -0.22917523980140686,
      "step": 174,
      "tokens/chosen_response_len_mean": 307.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 307.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.8390846969584946,
      "learning_rate": 4.97540899240276e-07,
      "logits/policy_chosen": -2.6149544715881348,
      "logits/policy_rejected": -2.7186009883880615,
      "logits/reference_chosen": -2.755988597869873,
      "logits/reference_rejected": -2.8059897422790527,
      "logps/policy_chosen": -83.78113555908203,
      "logps/policy_rejected": -158.60336303710938,
      "logps/reference_chosen": -105.53045654296875,
      "logps/reference_rejected": -161.6735076904297,
      "loss": 0.5707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2174932360649109,
      "rewards/margins": 0.18679174780845642,
      "rewards/rejected": 0.030701473355293274,
      "step": 175,
      "tokens/chosen_response_len_mean": 227.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 221.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 1.928131402745428,
      "learning_rate": 4.974417213499681e-07,
      "logits/policy_chosen": -2.97554874420166,
      "logits/policy_rejected": -3.0169177055358887,
      "logits/reference_chosen": -3.1351237297058105,
      "logits/reference_rejected": -3.054442882537842,
      "logps/policy_chosen": -155.18731689453125,
      "logps/policy_rejected": -306.79449462890625,
      "logps/reference_chosen": -198.6287841796875,
      "logps/reference_rejected": -296.349609375,
      "loss": 0.5884,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4344147741794586,
      "rewards/margins": 0.5388633608818054,
      "rewards/rejected": -0.1044485792517662,
      "step": 176,
      "tokens/chosen_response_len_mean": 399.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 380.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.5891349823773604,
      "learning_rate": 4.973405931111844e-07,
      "logits/policy_chosen": -2.475595474243164,
      "logits/policy_rejected": -2.5474050045013428,
      "logits/reference_chosen": -2.574971914291382,
      "logits/reference_rejected": -2.7303850650787354,
      "logps/policy_chosen": -60.937068939208984,
      "logps/policy_rejected": -196.1728515625,
      "logps/reference_chosen": -93.98703002929688,
      "logps/reference_rejected": -193.26068115234375,
      "loss": 0.5721,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.330499529838562,
      "rewards/margins": 0.3596211075782776,
      "rewards/rejected": -0.029121577739715576,
      "step": 177,
      "tokens/chosen_response_len_mean": 277.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 290.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 3.186008328730442,
      "learning_rate": 4.972375153210227e-07,
      "logits/policy_chosen": -2.73168683052063,
      "logits/policy_rejected": -2.80218505859375,
      "logits/reference_chosen": -2.8586461544036865,
      "logits/reference_rejected": -2.856407642364502,
      "logps/policy_chosen": -41.751312255859375,
      "logps/policy_rejected": -121.0931396484375,
      "logps/reference_chosen": -52.108253479003906,
      "logps/reference_rejected": -104.97093200683594,
      "loss": 0.5198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1035693883895874,
      "rewards/margins": 0.26479148864746094,
      "rewards/rejected": -0.16122210025787354,
      "step": 178,
      "tokens/chosen_response_len_mean": 131.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.4038166772062186,
      "learning_rate": 4.971324887919477e-07,
      "logits/policy_chosen": -2.768160820007324,
      "logits/policy_rejected": -2.2585337162017822,
      "logits/reference_chosen": -2.8858625888824463,
      "logits/reference_rejected": -2.4065427780151367,
      "logps/policy_chosen": -67.46320343017578,
      "logps/policy_rejected": -281.3670654296875,
      "logps/reference_chosen": -94.11170196533203,
      "logps/reference_rejected": -263.22698974609375,
      "loss": 0.5596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26648497581481934,
      "rewards/margins": 0.4478859305381775,
      "rewards/rejected": -0.18140093982219696,
      "step": 179,
      "tokens/chosen_response_len_mean": 239.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 333.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.530913530757402,
      "learning_rate": 4.970255143517838e-07,
      "logits/policy_chosen": -2.6580893993377686,
      "logits/policy_rejected": -2.717331886291504,
      "logits/reference_chosen": -2.90189790725708,
      "logits/reference_rejected": -2.8899142742156982,
      "logps/policy_chosen": -129.58033752441406,
      "logps/policy_rejected": -165.34884643554688,
      "logps/reference_chosen": -179.34756469726562,
      "logps/reference_rejected": -170.3519287109375,
      "loss": 0.5522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4976721405982971,
      "rewards/margins": 0.4476412534713745,
      "rewards/rejected": 0.050030872225761414,
      "step": 180,
      "tokens/chosen_response_len_mean": 500.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 318.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.633909183828016,
      "learning_rate": 4.969165928437091e-07,
      "logits/policy_chosen": -2.570159912109375,
      "logits/policy_rejected": -2.5334911346435547,
      "logits/reference_chosen": -2.760227680206299,
      "logits/reference_rejected": -2.3865246772766113,
      "logps/policy_chosen": -55.27775955200195,
      "logps/policy_rejected": -119.49053955078125,
      "logps/reference_chosen": -72.7205581665039,
      "logps/reference_rejected": -111.53352355957031,
      "loss": 0.5199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17442797124385834,
      "rewards/margins": 0.2539982497692108,
      "rewards/rejected": -0.07957027852535248,
      "step": 181,
      "tokens/chosen_response_len_mean": 307.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 267.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.3888599137710167,
      "learning_rate": 4.968057251262486e-07,
      "logits/policy_chosen": -2.6059675216674805,
      "logits/policy_rejected": -2.395353078842163,
      "logits/reference_chosen": -2.9607644081115723,
      "logits/reference_rejected": -1.8757669925689697,
      "logps/policy_chosen": -74.94976043701172,
      "logps/policy_rejected": -134.52859497070312,
      "logps/reference_chosen": -111.65763092041016,
      "logps/reference_rejected": -96.27473449707031,
      "loss": 0.5536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.36707866191864014,
      "rewards/margins": 0.7496173977851868,
      "rewards/rejected": -0.38253867626190186,
      "step": 182,
      "tokens/chosen_response_len_mean": 351.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 148.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.826021624132104,
      "learning_rate": 4.966929120732675e-07,
      "logits/policy_chosen": -2.3233590126037598,
      "logits/policy_rejected": -2.4008846282958984,
      "logits/reference_chosen": -2.5622141361236572,
      "logits/reference_rejected": -2.381047248840332,
      "logps/policy_chosen": -185.09976196289062,
      "logps/policy_rejected": -336.95941162109375,
      "logps/reference_chosen": -236.7534637451172,
      "logps/reference_rejected": -320.8805847167969,
      "loss": 0.5087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.516537070274353,
      "rewards/margins": 0.6773253679275513,
      "rewards/rejected": -0.16078826785087585,
      "step": 183,
      "tokens/chosen_response_len_mean": 441.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 384.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.26864738127188,
      "learning_rate": 4.965781545739638e-07,
      "logits/policy_chosen": -2.7422099113464355,
      "logits/policy_rejected": -2.7613723278045654,
      "logits/reference_chosen": -2.9637043476104736,
      "logits/reference_rejected": -2.921086549758911,
      "logps/policy_chosen": -45.85551452636719,
      "logps/policy_rejected": -118.8583984375,
      "logps/reference_chosen": -65.04806518554688,
      "logps/reference_rejected": -119.62251281738281,
      "loss": 0.5676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.191925510764122,
      "rewards/margins": 0.1842842549085617,
      "rewards/rejected": 0.007641255855560303,
      "step": 184,
      "tokens/chosen_response_len_mean": 266.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 242.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.8182732561665023,
      "learning_rate": 4.964614535328625e-07,
      "logits/policy_chosen": -2.523998975753784,
      "logits/policy_rejected": -2.4280190467834473,
      "logits/reference_chosen": -2.717611789703369,
      "logits/reference_rejected": -2.5818982124328613,
      "logps/policy_chosen": -250.7300262451172,
      "logps/policy_rejected": -131.51202392578125,
      "logps/reference_chosen": -304.0659484863281,
      "logps/reference_rejected": -136.93946838378906,
      "loss": 0.5284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.533359169960022,
      "rewards/margins": 0.47908464074134827,
      "rewards/rejected": 0.0542745515704155,
      "step": 185,
      "tokens/chosen_response_len_mean": 653.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 300.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.658487252908494,
      "learning_rate": 4.963428098698069e-07,
      "logits/policy_chosen": -2.8848061561584473,
      "logits/policy_rejected": -2.887396812438965,
      "logits/reference_chosen": -3.0770082473754883,
      "logits/reference_rejected": -3.012545347213745,
      "logps/policy_chosen": -26.844032287597656,
      "logps/policy_rejected": -64.93582916259766,
      "logps/reference_chosen": -37.33853530883789,
      "logps/reference_rejected": -57.781166076660156,
      "loss": 0.5617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10494504123926163,
      "rewards/margins": 0.17649173736572266,
      "rewards/rejected": -0.07154669612646103,
      "step": 186,
      "tokens/chosen_response_len_mean": 101.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 99.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.192921534395214,
      "learning_rate": 4.962222245199527e-07,
      "logits/policy_chosen": -2.5711615085601807,
      "logits/policy_rejected": -2.556881904602051,
      "logits/reference_chosen": -2.9153711795806885,
      "logits/reference_rejected": -2.4689133167266846,
      "logps/policy_chosen": -79.51646423339844,
      "logps/policy_rejected": -212.90431213378906,
      "logps/reference_chosen": -108.06434631347656,
      "logps/reference_rejected": -189.78848266601562,
      "loss": 0.5117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.285478800535202,
      "rewards/margins": 0.5166370868682861,
      "rewards/rejected": -0.2311583161354065,
      "step": 187,
      "tokens/chosen_response_len_mean": 368.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 330.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.185300778929063,
      "learning_rate": 4.960996984337598e-07,
      "logits/policy_chosen": -2.322800397872925,
      "logits/policy_rejected": -2.180086851119995,
      "logits/reference_chosen": -2.647416830062866,
      "logits/reference_rejected": -2.404383420944214,
      "logps/policy_chosen": -88.6377944946289,
      "logps/policy_rejected": -154.1780242919922,
      "logps/reference_chosen": -146.73565673828125,
      "logps/reference_rejected": -158.4337921142578,
      "loss": 0.4879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5809786319732666,
      "rewards/margins": 0.5384209156036377,
      "rewards/rejected": 0.042557742446660995,
      "step": 188,
      "tokens/chosen_response_len_mean": 496.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 375.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.018235573859185,
      "learning_rate": 4.959752325769852e-07,
      "logits/policy_chosen": -2.469785213470459,
      "logits/policy_rejected": -2.580235481262207,
      "logits/reference_chosen": -2.680108070373535,
      "logits/reference_rejected": -2.8124053478240967,
      "logps/policy_chosen": -57.80915069580078,
      "logps/policy_rejected": -210.04135131835938,
      "logps/reference_chosen": -90.7098617553711,
      "logps/reference_rejected": -191.3103485107422,
      "loss": 0.5062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.329007089138031,
      "rewards/margins": 0.5163170099258423,
      "rewards/rejected": -0.18730990588665009,
      "step": 189,
      "tokens/chosen_response_len_mean": 332.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 348.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.1505949518170437,
      "learning_rate": 4.958488279306753e-07,
      "logits/policy_chosen": -2.3762245178222656,
      "logits/policy_rejected": -2.466752767562866,
      "logits/reference_chosen": -2.847898483276367,
      "logits/reference_rejected": -2.2204389572143555,
      "logps/policy_chosen": -104.57975006103516,
      "logps/policy_rejected": -250.35414123535156,
      "logps/reference_chosen": -159.56100463867188,
      "logps/reference_rejected": -225.86146545410156,
      "loss": 0.522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5498126149177551,
      "rewards/margins": 0.7947396039962769,
      "rewards/rejected": -0.24492698907852173,
      "step": 190,
      "tokens/chosen_response_len_mean": 512.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 435.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.056135297935867,
      "learning_rate": 4.957204854911578e-07,
      "logits/policy_chosen": -2.485416889190674,
      "logits/policy_rejected": -2.3778135776519775,
      "logits/reference_chosen": -2.856623411178589,
      "logits/reference_rejected": -2.3928959369659424,
      "logps/policy_chosen": -83.4710693359375,
      "logps/policy_rejected": -267.17791748046875,
      "logps/reference_chosen": -124.20071411132812,
      "logps/reference_rejected": -209.21670532226562,
      "loss": 0.5036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40729638934135437,
      "rewards/margins": 0.9869081377983093,
      "rewards/rejected": -0.5796117782592773,
      "step": 191,
      "tokens/chosen_response_len_mean": 528.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 410.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.503765131980355,
      "learning_rate": 4.955902062700349e-07,
      "logits/policy_chosen": -2.3622069358825684,
      "logits/policy_rejected": -2.4439258575439453,
      "logits/reference_chosen": -2.7395248413085938,
      "logits/reference_rejected": -2.6338257789611816,
      "logps/policy_chosen": -76.7771224975586,
      "logps/policy_rejected": -262.30841064453125,
      "logps/reference_chosen": -117.23094940185547,
      "logps/reference_rejected": -224.7442626953125,
      "loss": 0.5445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40453827381134033,
      "rewards/margins": 0.7801797986030579,
      "rewards/rejected": -0.37564149498939514,
      "step": 192,
      "tokens/chosen_response_len_mean": 359.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 367.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.9703660280881734,
      "learning_rate": 4.954579912941737e-07,
      "logits/policy_chosen": -2.622279644012451,
      "logits/policy_rejected": -2.6470251083374023,
      "logits/reference_chosen": -2.853696823120117,
      "logits/reference_rejected": -2.840925931930542,
      "logps/policy_chosen": -38.201744079589844,
      "logps/policy_rejected": -138.85470581054688,
      "logps/reference_chosen": -63.35798645019531,
      "logps/reference_rejected": -113.46359252929688,
      "loss": 0.4864,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2515623867511749,
      "rewards/margins": 0.5054734349250793,
      "rewards/rejected": -0.25391101837158203,
      "step": 193,
      "tokens/chosen_response_len_mean": 157.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 169.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.9246186619159347,
      "learning_rate": 4.953238416057e-07,
      "logits/policy_chosen": -2.605665683746338,
      "logits/policy_rejected": -2.5566539764404297,
      "logits/reference_chosen": -2.9360122680664062,
      "logits/reference_rejected": -2.068471908569336,
      "logps/policy_chosen": -84.89468383789062,
      "logps/policy_rejected": -218.09048461914062,
      "logps/reference_chosen": -133.76991271972656,
      "logps/reference_rejected": -183.8456573486328,
      "loss": 0.4881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48875218629837036,
      "rewards/margins": 0.8312005996704102,
      "rewards/rejected": -0.3424483835697174,
      "step": 194,
      "tokens/chosen_response_len_mean": 360.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 238.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.80153859644754,
      "learning_rate": 4.951877582619881e-07,
      "logits/policy_chosen": -2.8731517791748047,
      "logits/policy_rejected": -2.975919246673584,
      "logits/reference_chosen": -3.0668981075286865,
      "logits/reference_rejected": -3.1014294624328613,
      "logps/policy_chosen": -24.165613174438477,
      "logps/policy_rejected": -61.2584342956543,
      "logps/reference_chosen": -23.939205169677734,
      "logps/reference_rejected": -43.215232849121094,
      "loss": 0.5741,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.002264055423438549,
      "rewards/margins": 0.1781679093837738,
      "rewards/rejected": -0.18043196201324463,
      "step": 195,
      "tokens/chosen_response_len_mean": 62.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 63.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.7527879974178955,
      "learning_rate": 4.950497423356542e-07,
      "logits/policy_chosen": -2.9940450191497803,
      "logits/policy_rejected": -2.710885763168335,
      "logits/reference_chosen": -3.169459581375122,
      "logits/reference_rejected": -2.7128515243530273,
      "logps/policy_chosen": -120.05692291259766,
      "logps/policy_rejected": -105.24632263183594,
      "logps/reference_chosen": -160.91629028320312,
      "logps/reference_rejected": -78.08905029296875,
      "loss": 0.5115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4085935354232788,
      "rewards/margins": 0.6801662445068359,
      "rewards/rejected": -0.27157267928123474,
      "step": 196,
      "tokens/chosen_response_len_mean": 332.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 53.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.473954328567208,
      "learning_rate": 4.949097949145471e-07,
      "logits/policy_chosen": -2.994095802307129,
      "logits/policy_rejected": -2.9911117553710938,
      "logits/reference_chosen": -3.0934770107269287,
      "logits/reference_rejected": -3.05942440032959,
      "logps/policy_chosen": -54.1805419921875,
      "logps/policy_rejected": -158.96043395996094,
      "logps/reference_chosen": -74.52861785888672,
      "logps/reference_rejected": -128.6375732421875,
      "loss": 0.4544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2034807801246643,
      "rewards/margins": 0.5067094564437866,
      "rewards/rejected": -0.3032286465167999,
      "step": 197,
      "tokens/chosen_response_len_mean": 182.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 165.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.6028329307586944,
      "learning_rate": 4.947679171017394e-07,
      "logits/policy_chosen": -2.7241291999816895,
      "logits/policy_rejected": -2.8773388862609863,
      "logits/reference_chosen": -2.9862842559814453,
      "logits/reference_rejected": -2.9953157901763916,
      "logps/policy_chosen": -56.624752044677734,
      "logps/policy_rejected": -255.8826141357422,
      "logps/reference_chosen": -101.90193176269531,
      "logps/reference_rejected": -203.60231018066406,
      "loss": 0.491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4527718722820282,
      "rewards/margins": 0.9755749106407166,
      "rewards/rejected": -0.5228029489517212,
      "step": 198,
      "tokens/chosen_response_len_mean": 225.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 217.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.8264702439738474,
      "learning_rate": 4.946241100155197e-07,
      "logits/policy_chosen": -2.6785213947296143,
      "logits/policy_rejected": -2.800018310546875,
      "logits/reference_chosen": -2.9413294792175293,
      "logits/reference_rejected": -2.913431406021118,
      "logps/policy_chosen": -38.42972183227539,
      "logps/policy_rejected": -164.37850952148438,
      "logps/reference_chosen": -65.64104461669922,
      "logps/reference_rejected": -128.12234497070312,
      "loss": 0.4973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27211323380470276,
      "rewards/margins": 0.6346749067306519,
      "rewards/rejected": -0.3625616431236267,
      "step": 199,
      "tokens/chosen_response_len_mean": 312.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 278.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.0030443321611644,
      "learning_rate": 4.944783747893825e-07,
      "logits/policy_chosen": -2.727503538131714,
      "logits/policy_rejected": -2.6386661529541016,
      "logits/reference_chosen": -2.9862632751464844,
      "logits/reference_rejected": -2.1630351543426514,
      "logps/policy_chosen": -84.61045837402344,
      "logps/policy_rejected": -191.14212036132812,
      "logps/reference_chosen": -125.34469604492188,
      "logps/reference_rejected": -150.1243896484375,
      "loss": 0.4714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4073423147201538,
      "rewards/margins": 0.8175195455551147,
      "rewards/rejected": -0.41017720103263855,
      "step": 200,
      "tokens/chosen_response_len_mean": 264.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 164.0,
      "tokens/rejected_response_len_std": 0.0
    }
  ],
  "logging_steps": 1,
  "max_steps": 1244,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}