{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984301412872841,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.006279434850863423,
      "grad_norm": 2389741299.6785245,
      "learning_rate": 3.125e-08,
      "logits/chosen": 0.9376043081283569,
      "logits/rejected": 1.0501841306686401,
      "logps/chosen": -447.9299011230469,
      "logps/pi_response": -306.161865234375,
      "logps/ref_response": -306.4182434082031,
      "logps/rejected": -457.8988952636719,
      "loss": 0.6928,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0020125629380345345,
      "rewards/margins": 0.0018250466091558337,
      "rewards/rejected": 0.00018751618335954845,
      "step": 1
    },
    {
      "epoch": 0.06279434850863422,
      "grad_norm": 0.0,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": 0.12129918485879898,
      "logits/rejected": 0.1437881588935852,
      "logps/chosen": -3537.467041015625,
      "logps/pi_response": -4150.1064453125,
      "logps/ref_response": -316.7183837890625,
      "logps/rejected": -3190.5419921875,
      "loss": 10.1423,
      "rewards/accuracies": 0.4930555522441864,
      "rewards/chosen": -31.1226806640625,
      "rewards/margins": -3.1455698013305664,
      "rewards/rejected": -27.977109909057617,
      "step": 10
    },
    {
      "epoch": 0.12558869701726844,
      "grad_norm": 0.0,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3586.97607421875,
      "logps/pi_response": -4675.82373046875,
      "logps/ref_response": -314.9863586425781,
      "logps/rejected": -3205.0859375,
      "loss": 10.1147,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -31.944461822509766,
      "rewards/margins": -3.5670688152313232,
      "rewards/rejected": -28.377389907836914,
      "step": 20
    },
    {
      "epoch": 0.18838304552590268,
      "grad_norm": 0.0,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3549.117919921875,
      "logps/pi_response": -4436.2509765625,
      "logps/ref_response": -306.4608459472656,
      "logps/rejected": -3030.349853515625,
      "loss": 10.9098,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -31.402786254882812,
      "rewards/margins": -4.936440467834473,
      "rewards/rejected": -26.46634292602539,
      "step": 30
    },
    {
      "epoch": 0.25117739403453687,
      "grad_norm": 0.0,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3818.757080078125,
      "logps/pi_response": -4928.62890625,
      "logps/ref_response": -323.02288818359375,
      "logps/rejected": -3543.67822265625,
      "loss": 11.076,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -33.8750114440918,
      "rewards/margins": -2.380995273590088,
      "rewards/rejected": -31.4940128326416,
      "step": 40
    },
    {
      "epoch": 0.3139717425431711,
      "grad_norm": 0.0,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -4011.28271484375,
      "logps/pi_response": -5043.599609375,
      "logps/ref_response": -320.0797424316406,
      "logps/rejected": -3335.053466796875,
      "loss": 10.8383,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -35.91133499145508,
      "rewards/margins": -6.445613861083984,
      "rewards/rejected": -29.46571922302246,
      "step": 50
    },
    {
      "epoch": 0.37676609105180536,
      "grad_norm": 0.0,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3751.494140625,
      "logps/pi_response": -4971.9267578125,
      "logps/ref_response": -311.6258544921875,
      "logps/rejected": -3540.6640625,
      "loss": 10.2301,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -33.251548767089844,
      "rewards/margins": -1.7325384616851807,
      "rewards/rejected": -31.519006729125977,
      "step": 60
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 0.0,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3980.702392578125,
      "logps/pi_response": -5090.5732421875,
      "logps/ref_response": -329.1057434082031,
      "logps/rejected": -3480.60595703125,
      "loss": 10.855,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -35.667266845703125,
      "rewards/margins": -4.873533725738525,
      "rewards/rejected": -30.793731689453125,
      "step": 70
    },
    {
      "epoch": 0.5023547880690737,
      "grad_norm": 0.0,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3804.789794921875,
      "logps/pi_response": -4688.3935546875,
      "logps/ref_response": -309.07379150390625,
      "logps/rejected": -3321.96923828125,
      "loss": 10.0252,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -34.202884674072266,
      "rewards/margins": -4.780238628387451,
      "rewards/rejected": -29.42264175415039,
      "step": 80
    },
    {
      "epoch": 0.565149136577708,
      "grad_norm": 0.0,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3998.41845703125,
      "logps/pi_response": -5007.4326171875,
      "logps/ref_response": -320.7072448730469,
      "logps/rejected": -3723.192626953125,
      "loss": 10.9442,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -35.741825103759766,
      "rewards/margins": -2.8130385875701904,
      "rewards/rejected": -32.92878723144531,
      "step": 90
    },
    {
      "epoch": 0.6279434850863422,
      "grad_norm": 0.0,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3574.7734375,
      "logps/pi_response": -4309.73779296875,
      "logps/ref_response": -297.6278076171875,
      "logps/rejected": -2943.09228515625,
      "loss": 11.0889,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -31.873769760131836,
      "rewards/margins": -6.0053815841674805,
      "rewards/rejected": -25.868383407592773,
      "step": 100
    },
    {
      "epoch": 0.6907378335949764,
      "grad_norm": 0.0,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3946.592529296875,
      "logps/pi_response": -4776.75439453125,
      "logps/ref_response": -326.58782958984375,
      "logps/rejected": -3317.11669921875,
      "loss": 10.8242,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -35.36956024169922,
      "rewards/margins": -6.115808486938477,
      "rewards/rejected": -29.25374984741211,
      "step": 110
    },
    {
      "epoch": 0.7535321821036107,
      "grad_norm": 0.0,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3573.376953125,
      "logps/pi_response": -4478.4462890625,
      "logps/ref_response": -297.75628662109375,
      "logps/rejected": -3231.69677734375,
      "loss": 10.9182,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -31.708187103271484,
      "rewards/margins": -3.1025025844573975,
      "rewards/rejected": -28.60568618774414,
      "step": 120
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 0.0,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3555.366455078125,
      "logps/pi_response": -4567.4677734375,
      "logps/ref_response": -308.615966796875,
      "logps/rejected": -3327.48193359375,
      "loss": 10.3728,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -31.474365234375,
      "rewards/margins": -2.0411860942840576,
      "rewards/rejected": -29.433177947998047,
      "step": 130
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 0.0,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3977.247314453125,
      "logps/pi_response": -4856.95458984375,
      "logps/ref_response": -321.81634521484375,
      "logps/rejected": -3193.8388671875,
      "loss": 10.413,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -35.818058013916016,
      "rewards/margins": -7.793890953063965,
      "rewards/rejected": -28.024166107177734,
      "step": 140
    },
    {
      "epoch": 0.9419152276295133,
      "grad_norm": 0.0,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": 0.0,
      "logits/rejected": 0.0,
      "logps/chosen": -3445.246826171875,
      "logps/pi_response": -4451.90869140625,
      "logps/ref_response": -298.1885986328125,
      "logps/rejected": -3225.22802734375,
      "loss": 10.872,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -30.63448715209961,
      "rewards/margins": -1.953590750694275,
      "rewards/rejected": -28.680896759033203,
      "step": 150
    },
    {
      "epoch": 0.9984301412872841,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 10.661155712679497,
      "train_runtime": 6456.3708,
      "train_samples_per_second": 3.156,
      "train_steps_per_second": 0.025
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}