{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9968652037617555,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "learning_rate": 3.125e-08,
      "logits/chosen": -2.6344056129455566,
      "logits/rejected": -2.5906338691711426,
      "logps/chosen": -158.64126586914062,
      "logps/pi_response": -86.09246826171875,
      "logps/ref_response": -86.09246826171875,
      "logps/rejected": -148.42047119140625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.691999673843384,
      "logits/rejected": -2.6960792541503906,
      "logps/chosen": -233.6039276123047,
      "logps/pi_response": -119.84125518798828,
      "logps/ref_response": -118.89913940429688,
      "logps/rejected": -252.53822326660156,
      "loss": 0.692,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012611959129571915,
      "rewards/margins": 0.0004196475783828646,
      "rewards/rejected": -0.013031606562435627,
      "step": 10
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -2.6983652114868164,
      "logits/rejected": -2.669889211654663,
      "logps/chosen": -250.45095825195312,
      "logps/pi_response": -146.63462829589844,
      "logps/ref_response": -131.12376403808594,
      "logps/rejected": -277.07781982421875,
      "loss": 0.6793,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.18149897456169128,
      "rewards/margins": 0.05194617435336113,
      "rewards/rejected": -0.23344513773918152,
      "step": 20
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -2.5047459602355957,
      "logits/rejected": -2.444232940673828,
      "logps/chosen": -243.5947265625,
      "logps/pi_response": -132.4476318359375,
      "logps/ref_response": -113.8835220336914,
      "logps/rejected": -249.1999969482422,
      "loss": 0.667,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1728166788816452,
      "rewards/margins": 0.11773113906383514,
      "rewards/rejected": -0.29054778814315796,
      "step": 30
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -2.314911365509033,
      "logits/rejected": -2.292646884918213,
      "logps/chosen": -254.43429565429688,
      "logps/pi_response": -164.00875854492188,
      "logps/ref_response": -110.53662109375,
      "logps/rejected": -328.1941833496094,
      "loss": 0.6434,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.5214705467224121,
      "rewards/margins": 0.17735092341899872,
      "rewards/rejected": -0.698821485042572,
      "step": 40
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -2.232452392578125,
      "logits/rejected": -2.202117443084717,
      "logps/chosen": -283.18255615234375,
      "logps/pi_response": -181.59396362304688,
      "logps/ref_response": -120.1254653930664,
      "logps/rejected": -310.2931213378906,
      "loss": 0.6299,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.5738414525985718,
      "rewards/margins": 0.15463915467262268,
      "rewards/rejected": -0.7284805178642273,
      "step": 50
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -2.1293296813964844,
      "logits/rejected": -2.094069004058838,
      "logps/chosen": -288.9293212890625,
      "logps/pi_response": -199.8803253173828,
      "logps/ref_response": -107.5267333984375,
      "logps/rejected": -332.07403564453125,
      "loss": 0.6242,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7527658939361572,
      "rewards/margins": 0.31268495321273804,
      "rewards/rejected": -1.0654507875442505,
      "step": 60
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -2.1092419624328613,
      "logits/rejected": -2.055610418319702,
      "logps/chosen": -297.887451171875,
      "logps/pi_response": -207.2319793701172,
      "logps/ref_response": -112.31196594238281,
      "logps/rejected": -314.06011962890625,
      "loss": 0.6006,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.7968708276748657,
      "rewards/margins": 0.35347780585289,
      "rewards/rejected": -1.150348424911499,
      "step": 70
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -2.1043503284454346,
      "logits/rejected": -2.0406947135925293,
      "logps/chosen": -277.8828430175781,
      "logps/pi_response": -199.2911376953125,
      "logps/ref_response": -99.59819793701172,
      "logps/rejected": -338.6634826660156,
      "loss": 0.6184,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.8253669738769531,
      "rewards/margins": 0.35502859950065613,
      "rewards/rejected": -1.1803953647613525,
      "step": 80
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -2.1398611068725586,
      "logits/rejected": -2.0961031913757324,
      "logps/chosen": -299.2795715332031,
      "logps/pi_response": -207.5061798095703,
      "logps/ref_response": -100.86174011230469,
      "logps/rejected": -342.63848876953125,
      "loss": 0.6061,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.9135547876358032,
      "rewards/margins": 0.3614444434642792,
      "rewards/rejected": -1.2749992609024048,
      "step": 90
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -2.041874647140503,
      "logits/rejected": -2.030878782272339,
      "logps/chosen": -306.0923767089844,
      "logps/pi_response": -226.0547332763672,
      "logps/ref_response": -112.6884765625,
      "logps/rejected": -394.3197326660156,
      "loss": 0.5803,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9776135683059692,
      "rewards/margins": 0.42801570892333984,
      "rewards/rejected": -1.4056293964385986,
      "step": 100
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -2.08709979057312,
      "logits/rejected": -2.1020517349243164,
      "logps/chosen": -311.7897033691406,
      "logps/pi_response": -208.67361450195312,
      "logps/ref_response": -100.60249328613281,
      "logps/rejected": -340.69842529296875,
      "loss": 0.6037,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.9508616328239441,
      "rewards/margins": 0.34958645701408386,
      "rewards/rejected": -1.3004481792449951,
      "step": 110
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -2.1009232997894287,
      "logits/rejected": -2.0271968841552734,
      "logps/chosen": -340.5748596191406,
      "logps/pi_response": -246.27499389648438,
      "logps/ref_response": -130.39340209960938,
      "logps/rejected": -368.299560546875,
      "loss": 0.5991,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.0281145572662354,
      "rewards/margins": 0.2725009322166443,
      "rewards/rejected": -1.3006153106689453,
      "step": 120
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -2.131903886795044,
      "logits/rejected": -2.1143031120300293,
      "logps/chosen": -329.1007995605469,
      "logps/pi_response": -242.9829559326172,
      "logps/ref_response": -113.8158950805664,
      "logps/rejected": -387.0042419433594,
      "loss": 0.5844,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.1540590524673462,
      "rewards/margins": 0.3786469101905823,
      "rewards/rejected": -1.5327059030532837,
      "step": 130
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -2.1207737922668457,
      "logits/rejected": -2.0526301860809326,
      "logps/chosen": -363.8649597167969,
      "logps/pi_response": -283.3131408691406,
      "logps/ref_response": -119.49796295166016,
      "logps/rejected": -411.9598083496094,
      "loss": 0.5928,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.3548119068145752,
      "rewards/margins": 0.4335576891899109,
      "rewards/rejected": -1.7883697748184204,
      "step": 140
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -2.069753646850586,
      "logits/rejected": -2.039280652999878,
      "logps/chosen": -339.0675354003906,
      "logps/pi_response": -243.1265869140625,
      "logps/ref_response": -101.72931671142578,
      "logps/rejected": -374.07086181640625,
      "loss": 0.5747,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2384488582611084,
      "rewards/margins": 0.3813491463661194,
      "rewards/rejected": -1.6197980642318726,
      "step": 150
    },
    {
      "epoch": 1.0,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 0.6180852314211288,
      "train_runtime": 4551.3166,
      "train_samples_per_second": 4.477,
      "train_steps_per_second": 0.035
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}