{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.99581589958159,
  "eval_steps": 500,
  "global_step": 119,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "learning_rate": 4.166666666666666e-08,
      "logits/chosen": -2.7076048851013184,
      "logits/rejected": -2.5675482749938965,
      "logps/chosen": -287.144287109375,
      "logps/pi_response": -67.09939575195312,
      "logps/ref_response": -67.09939575195312,
      "logps/rejected": -200.97291564941406,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.725297689437866,
      "logits/rejected": -2.6898391246795654,
      "logps/chosen": -235.20797729492188,
      "logps/pi_response": -75.70732116699219,
      "logps/ref_response": -75.67623901367188,
      "logps/rejected": -201.98898315429688,
      "loss": 0.6924,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": 0.001756811747327447,
      "rewards/margins": 0.0011220998130738735,
      "rewards/rejected": 0.0006347117596305907,
      "step": 10
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931352528237397e-07,
      "logits/chosen": -2.7490615844726562,
      "logits/rejected": -2.7063941955566406,
      "logps/chosen": -246.2781982421875,
      "logps/pi_response": -75.30223083496094,
      "logps/ref_response": -72.46954345703125,
      "logps/rejected": -174.66903686523438,
      "loss": 0.6749,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03482687845826149,
      "rewards/margins": 0.037110961973667145,
      "rewards/rejected": -0.0022840884048491716,
      "step": 20
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.658920803689553e-07,
      "logits/chosen": -2.6719274520874023,
      "logits/rejected": -2.637406826019287,
      "logps/chosen": -231.8843536376953,
      "logps/pi_response": -95.55384826660156,
      "logps/ref_response": -74.0744400024414,
      "logps/rejected": -167.51046752929688,
      "loss": 0.6401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03914440795779228,
      "rewards/margins": 0.12467870861291885,
      "rewards/rejected": -0.16382311284542084,
      "step": 30
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.201712553872657e-07,
      "logits/chosen": -2.6597397327423096,
      "logits/rejected": -2.6165637969970703,
      "logps/chosen": -259.60015869140625,
      "logps/pi_response": -130.00448608398438,
      "logps/ref_response": -74.10218048095703,
      "logps/rejected": -219.8263702392578,
      "loss": 0.6048,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1824723184108734,
      "rewards/margins": 0.2193528711795807,
      "rewards/rejected": -0.4018251895904541,
      "step": 40
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.598859066780754e-07,
      "logits/chosen": -2.6596665382385254,
      "logits/rejected": -2.6250228881835938,
      "logps/chosen": -300.73388671875,
      "logps/pi_response": -165.25631713867188,
      "logps/ref_response": -79.56159973144531,
      "logps/rejected": -233.9432830810547,
      "loss": 0.5723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26315242052078247,
      "rewards/margins": 0.41799718141555786,
      "rewards/rejected": -0.6811496019363403,
      "step": 50
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9019570347986706e-07,
      "logits/chosen": -2.637012004852295,
      "logits/rejected": -2.6089441776275635,
      "logps/chosen": -297.0741271972656,
      "logps/pi_response": -182.01626586914062,
      "logps/ref_response": -82.40287780761719,
      "logps/rejected": -261.73394775390625,
      "loss": 0.5474,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.37535199522972107,
      "rewards/margins": 0.5962287783622742,
      "rewards/rejected": -0.9715806841850281,
      "step": 60
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1706525253979534e-07,
      "logits/chosen": -2.679154634475708,
      "logits/rejected": -2.6408259868621826,
      "logps/chosen": -286.2748107910156,
      "logps/pi_response": -170.31283569335938,
      "logps/ref_response": -68.73023986816406,
      "logps/rejected": -258.6576843261719,
      "loss": 0.5323,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.48963016271591187,
      "rewards/margins": 0.44551196694374084,
      "rewards/rejected": -0.9351421594619751,
      "step": 70
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4675360263490295e-07,
      "logits/chosen": -2.6439719200134277,
      "logits/rejected": -2.6195614337921143,
      "logps/chosen": -255.16000366210938,
      "logps/pi_response": -177.17091369628906,
      "logps/ref_response": -68.83003997802734,
      "logps/rejected": -276.4742126464844,
      "loss": 0.5112,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.557772159576416,
      "rewards/margins": 0.548255205154419,
      "rewards/rejected": -1.106027364730835,
      "step": 80
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.527854855097224e-08,
      "logits/chosen": -2.7123055458068848,
      "logits/rejected": -2.67814302444458,
      "logps/chosen": -299.35333251953125,
      "logps/pi_response": -196.30416870117188,
      "logps/ref_response": -69.44734191894531,
      "logps/rejected": -288.25262451171875,
      "loss": 0.5049,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.6541774272918701,
      "rewards/margins": 0.6144094467163086,
      "rewards/rejected": -1.2685868740081787,
      "step": 90
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.790158337517127e-08,
      "logits/chosen": -2.6441962718963623,
      "logits/rejected": -2.614448070526123,
      "logps/chosen": -331.6165466308594,
      "logps/pi_response": -203.06991577148438,
      "logps/ref_response": -70.62646484375,
      "logps/rejected": -300.808349609375,
      "loss": 0.4971,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.6863395571708679,
      "rewards/margins": 0.574430525302887,
      "rewards/rejected": -1.2607699632644653,
      "step": 100
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.677580722139671e-09,
      "logits/chosen": -2.734614849090576,
      "logits/rejected": -2.6600518226623535,
      "logps/chosen": -334.9231872558594,
      "logps/pi_response": -206.056640625,
      "logps/ref_response": -80.19813537597656,
      "logps/rejected": -307.8163757324219,
      "loss": 0.5154,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5829799771308899,
      "rewards/margins": 0.7412832975387573,
      "rewards/rejected": -1.324263334274292,
      "step": 110
    },
    {
      "epoch": 1.0,
      "step": 119,
      "total_flos": 0.0,
      "train_loss": 0.5652808101237321,
      "train_runtime": 3576.3079,
      "train_samples_per_second": 4.273,
      "train_steps_per_second": 0.033
    }
  ],
  "logging_steps": 10,
  "max_steps": 119,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}