{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984301412872841,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.006279434850863423,
      "grad_norm": 5.603575760096964,
      "learning_rate": 3.125e-08,
      "logits/chosen": 0.18015038967132568,
      "logits/rejected": 0.2519298493862152,
      "logps/chosen": -297.10906982421875,
      "logps/pi_response": -130.58929443359375,
      "logps/ref_response": -130.58929443359375,
      "logps/rejected": -316.44769287109375,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06279434850863422,
      "grad_norm": 5.927013620517721,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": 0.16517189145088196,
      "logits/rejected": 0.31397953629493713,
      "logps/chosen": -243.69070434570312,
      "logps/pi_response": -120.14439392089844,
      "logps/ref_response": -120.15902709960938,
      "logps/rejected": -281.0929870605469,
      "loss": 0.6929,
      "rewards/accuracies": 0.5069444179534912,
      "rewards/chosen": 9.301294630859047e-05,
      "rewards/margins": 0.0011727283708751202,
      "rewards/rejected": -0.001079715322703123,
      "step": 10
    },
    {
      "epoch": 0.12558869701726844,
      "grad_norm": 6.040197286277514,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": 0.13175079226493835,
      "logits/rejected": 0.32193881273269653,
      "logps/chosen": -244.1867218017578,
      "logps/pi_response": -121.63621520996094,
      "logps/ref_response": -121.85536193847656,
      "logps/rejected": -266.6852722167969,
      "loss": 0.6886,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.01275191456079483,
      "rewards/margins": 0.0108140604570508,
      "rewards/rejected": -0.023565974086523056,
      "step": 20
    },
    {
      "epoch": 0.18838304552590268,
      "grad_norm": 6.198937762628843,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": 0.1714327037334442,
      "logits/rejected": 0.3029390871524811,
      "logps/chosen": -244.3697509765625,
      "logps/pi_response": -109.6906967163086,
      "logps/ref_response": -110.8894271850586,
      "logps/rejected": -290.1263732910156,
      "loss": 0.6686,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.042444389313459396,
      "rewards/margins": 0.055226124823093414,
      "rewards/rejected": -0.09767051041126251,
      "step": 30
    },
    {
      "epoch": 0.25117739403453687,
      "grad_norm": 5.780662683878471,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": 0.2108462154865265,
      "logits/rejected": 0.3984522521495819,
      "logps/chosen": -287.33648681640625,
      "logps/pi_response": -125.24674224853516,
      "logps/ref_response": -129.86325073242188,
      "logps/rejected": -316.37408447265625,
      "loss": 0.6347,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08861993253231049,
      "rewards/margins": 0.1344626396894455,
      "rewards/rejected": -0.22308258712291718,
      "step": 40
    },
    {
      "epoch": 0.3139717425431711,
      "grad_norm": 5.870253990249712,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": 0.37238794565200806,
      "logits/rejected": 0.5105336308479309,
      "logps/chosen": -246.9188232421875,
      "logps/pi_response": -108.8661117553711,
      "logps/ref_response": -116.5090560913086,
      "logps/rejected": -310.613037109375,
      "loss": 0.606,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.1104748398065567,
      "rewards/margins": 0.24330809712409973,
      "rewards/rejected": -0.3537829518318176,
      "step": 50
    },
    {
      "epoch": 0.37676609105180536,
      "grad_norm": 9.335664801309742,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": 0.5169380903244019,
      "logits/rejected": 0.6794592142105103,
      "logps/chosen": -267.8638000488281,
      "logps/pi_response": -116.31705474853516,
      "logps/ref_response": -119.4989242553711,
      "logps/rejected": -347.69482421875,
      "loss": 0.5813,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.19944007694721222,
      "rewards/margins": 0.4112206995487213,
      "rewards/rejected": -0.6106608510017395,
      "step": 60
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 7.251270500863869,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": 0.6026707887649536,
      "logits/rejected": 0.7696335911750793,
      "logps/chosen": -236.616455078125,
      "logps/pi_response": -114.6134262084961,
      "logps/ref_response": -116.70068359375,
      "logps/rejected": -368.1206359863281,
      "loss": 0.5567,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.262630432844162,
      "rewards/margins": 0.5536423921585083,
      "rewards/rejected": -0.8162728548049927,
      "step": 70
    },
    {
      "epoch": 0.5023547880690737,
      "grad_norm": 7.652733380703419,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": 0.4738255441188812,
      "logits/rejected": 0.8158077001571655,
      "logps/chosen": -325.28857421875,
      "logps/pi_response": -128.89028930664062,
      "logps/ref_response": -127.53900146484375,
      "logps/rejected": -348.1357116699219,
      "loss": 0.5848,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.36525511741638184,
      "rewards/margins": 0.3771258294582367,
      "rewards/rejected": -0.7423809766769409,
      "step": 80
    },
    {
      "epoch": 0.565149136577708,
      "grad_norm": 6.2380923795979655,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": 0.43513059616088867,
      "logits/rejected": 0.8127248883247375,
      "logps/chosen": -301.32684326171875,
      "logps/pi_response": -135.54782104492188,
      "logps/ref_response": -129.38760375976562,
      "logps/rejected": -420.8568420410156,
      "loss": 0.5435,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.29517459869384766,
      "rewards/margins": 0.6168515086174011,
      "rewards/rejected": -0.9120261073112488,
      "step": 90
    },
    {
      "epoch": 0.6279434850863422,
      "grad_norm": 7.580754321821852,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": 0.6774718165397644,
      "logits/rejected": 0.8504177331924438,
      "logps/chosen": -283.39569091796875,
      "logps/pi_response": -121.60555267333984,
      "logps/ref_response": -114.0061264038086,
      "logps/rejected": -304.47467041015625,
      "loss": 0.5426,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.3073732852935791,
      "rewards/margins": 0.3342594504356384,
      "rewards/rejected": -0.6416326761245728,
      "step": 100
    },
    {
      "epoch": 0.6907378335949764,
      "grad_norm": 7.058762821080314,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": 0.6427871584892273,
      "logits/rejected": 0.9476824998855591,
      "logps/chosen": -297.91217041015625,
      "logps/pi_response": -136.62130737304688,
      "logps/ref_response": -125.8144760131836,
      "logps/rejected": -383.70025634765625,
      "loss": 0.5314,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.33735179901123047,
      "rewards/margins": 0.5089818835258484,
      "rewards/rejected": -0.8463336825370789,
      "step": 110
    },
    {
      "epoch": 0.7535321821036107,
      "grad_norm": 7.301058922446218,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 0.5713673830032349,
      "logits/rejected": 0.8841819763183594,
      "logps/chosen": -320.47723388671875,
      "logps/pi_response": -131.65252685546875,
      "logps/ref_response": -120.58707427978516,
      "logps/rejected": -369.21661376953125,
      "loss": 0.5542,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39976996183395386,
      "rewards/margins": 0.5199070572853088,
      "rewards/rejected": -0.9196769595146179,
      "step": 120
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 6.728836632952595,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": 0.5993797779083252,
      "logits/rejected": 0.8266558647155762,
      "logps/chosen": -276.78338623046875,
      "logps/pi_response": -135.93252563476562,
      "logps/ref_response": -123.1449966430664,
      "logps/rejected": -372.402587890625,
      "loss": 0.5271,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3279780447483063,
      "rewards/margins": 0.5935764908790588,
      "rewards/rejected": -0.921554446220398,
      "step": 130
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 7.263275222680605,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": 0.5662352442741394,
      "logits/rejected": 0.8682255744934082,
      "logps/chosen": -283.9330139160156,
      "logps/pi_response": -130.8988037109375,
      "logps/ref_response": -121.63087463378906,
      "logps/rejected": -404.0865173339844,
      "loss": 0.5299,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3388724625110626,
      "rewards/margins": 0.7475099563598633,
      "rewards/rejected": -1.0863823890686035,
      "step": 140
    },
    {
      "epoch": 0.9419152276295133,
      "grad_norm": 7.996654815605861,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": 0.5043578147888184,
      "logits/rejected": 0.8957873582839966,
      "logps/chosen": -317.9568786621094,
      "logps/pi_response": -145.6123504638672,
      "logps/ref_response": -132.86119079589844,
      "logps/rejected": -398.7013854980469,
      "loss": 0.5193,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3905355930328369,
      "rewards/margins": 0.5843077898025513,
      "rewards/rejected": -0.9748433232307434,
      "step": 150
    },
    {
      "epoch": 0.9984301412872841,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 0.5789255676029613,
      "train_runtime": 4418.8006,
      "train_samples_per_second": 4.612,
      "train_steps_per_second": 0.036
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}