Model save

README.md

@@ -0,0 +1,74 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-dpo-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-dpo-full
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5714
+- Rewards/chosen: -0.4765
+- Rewards/rejected: -1.7868
+- Rewards/accuracies: 0.7619
+- Rewards/margins: 1.3103
+- Logps/rejected: -294.2741
+- Logps/chosen: -305.4321
+- Logits/rejected: -2.4275
+- Logits/chosen: -2.4841
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.5509        | 0.26  | 500  | 0.6311          | 0.8114         | -0.1181          | 0.7619             | 0.9294          | -277.5868      | -292.5533    | -2.4249         | -2.4869       |
+| 0.6191        | 0.52  | 1000 | 0.5984          | 0.0381         | -1.0554          | 0.7599             | 1.0935          | -286.9601      | -300.2859    | -2.4280         | -2.4907       |
+| 0.5342        | 0.78  | 1500 | 0.5819          | 0.0994         | -1.1560          | 0.7560             | 1.2554          | -287.9663      | -299.6732    | -2.4208         | -2.4793       |
+
+
+### Framework versions
+
+- Transformers 4.36.1
+- Pytorch 2.0.1+cu117
+- Datasets 2.16.1
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.484065532684326,
+    "eval_logits/rejected": -2.4274542331695557,
+    "eval_logps/chosen": -305.43212890625,
+    "eval_logps/rejected": -294.2740783691406,
+    "eval_loss": 0.5714402198791504,
+    "eval_rewards/accuracies": 0.761904776096344,
+    "eval_rewards/chosen": -0.4765053391456604,
+    "eval_rewards/margins": 1.3102855682373047,
+    "eval_rewards/rejected": -1.7867907285690308,
+    "eval_runtime": 613.7286,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 3.259,
+    "eval_steps_per_second": 0.103,
+    "train_loss": 0.5469513680846325,
+    "train_runtime": 31987.9403,
+    "train_samples": 61135,
+    "train_samples_per_second": 1.911,
+    "train_steps_per_second": 0.06
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.36.1",
+  "use_cache": false,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.484065532684326,
+    "eval_logits/rejected": -2.4274542331695557,
+    "eval_logps/chosen": -305.43212890625,
+    "eval_logps/rejected": -294.2740783691406,
+    "eval_loss": 0.5714402198791504,
+    "eval_rewards/accuracies": 0.761904776096344,
+    "eval_rewards/chosen": -0.4765053391456604,
+    "eval_rewards/margins": 1.3102855682373047,
+    "eval_rewards/rejected": -1.7867907285690308,
+    "eval_runtime": 613.7286,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 3.259,
+    "eval_steps_per_second": 0.103
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.1"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bed0946f3404fda76ec1b3c42687712a74cfa981f64b593183f5a0b1469da37c
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1f253b60e78de1283921bb8677fd5370e9e007b56e60a1ea2d4bc88b7ab1f01
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5e95d7209d2b8548bdf8edd75b7e9234913d01ee17c8f48d0a377646c241cac
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Jan21_00-47-39_node14/events.out.tfevents.1705769697.node14.3000533.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31655d75d8f112070605b567662aa7d861ce658b8fc21f256441c665b3f85f37
+size 11283

runs/Jan29_16-30-36_node01/events.out.tfevents.1706517112.node01.2477677.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fb85e0709a6893bb8aa4b3bc1c1ab6b56d9ac0042ae6886b88eafa3b717b32b
+size 5677

runs/Jan29_16-36-38_node01/events.out.tfevents.1706517422.node01.2480881.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8a5a77375b65bce27334caf381a210a85651696e9cc6651537bfce3b712e155
+size 128589

runs/Jan29_16-36-38_node01/events.out.tfevents.1706550024.node01.2480881.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a477d7cf45ad8760f92f4a34ac0c8895349180786b7ceede262f1a3257be077
+size 828

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 4096,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5469513680846325,
+    "train_runtime": 31987.9403,
+    "train_samples": 61135,
+    "train_samples_per_second": 1.911,
+    "train_steps_per_second": 0.06
+}

trainer_state.json

@@ -0,0 +1,2766 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1911,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6041666666666664e-09,
+      "logits/chosen": -2.463043451309204,
+      "logits/rejected": -2.288743019104004,
+      "logps/chosen": -301.1433410644531,
+      "logps/rejected": -128.25608825683594,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.6041666666666667e-08,
+      "logits/chosen": -2.5560247898101807,
+      "logits/rejected": -2.555253028869629,
+      "logps/chosen": -286.1558837890625,
+      "logps/rejected": -256.28131103515625,
+      "loss": 0.697,
+      "rewards/accuracies": 0.3333333432674408,
+      "rewards/chosen": -0.0025797931011766195,
+      "rewards/margins": -0.008782301098108292,
+      "rewards/rejected": 0.006202507298439741,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.208333333333333e-08,
+      "logits/chosen": -2.648404598236084,
+      "logits/rejected": -2.6275510787963867,
+      "logps/chosen": -316.64373779296875,
+      "logps/rejected": -314.99212646484375,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.0072450353763997555,
+      "rewards/margins": 0.0013799279695376754,
+      "rewards/rejected": -0.008624963462352753,
+      "step": 20
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 7.812499999999999e-08,
+      "logits/chosen": -2.683773994445801,
+      "logits/rejected": -2.550048828125,
+      "logps/chosen": -306.83282470703125,
+      "logps/rejected": -266.6906433105469,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.011949767358601093,
+      "rewards/margins": 0.020141970366239548,
+      "rewards/rejected": -0.008192205801606178,
+      "step": 30
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.634948968887329,
+      "logits/rejected": -2.5988709926605225,
+      "logps/chosen": -265.98968505859375,
+      "logps/rejected": -252.77835083007812,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.08117702603340149,
+      "rewards/margins": 0.05083204060792923,
+      "rewards/rejected": 0.03034498728811741,
+      "step": 40
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3020833333333334e-07,
+      "logits/chosen": -2.644986152648926,
+      "logits/rejected": -2.589542865753174,
+      "logps/chosen": -323.46881103515625,
+      "logps/rejected": -284.9610900878906,
+      "loss": 0.6542,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.2217252552509308,
+      "rewards/margins": 0.12405480444431305,
+      "rewards/rejected": 0.09767045080661774,
+      "step": 50
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": -2.599961280822754,
+      "logits/rejected": -2.491401195526123,
+      "logps/chosen": -305.62152099609375,
+      "logps/rejected": -236.73916625976562,
+      "loss": 0.6107,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.29835107922554016,
+      "rewards/margins": 0.23799380660057068,
+      "rewards/rejected": 0.06035725399851799,
+      "step": 60
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8229166666666666e-07,
+      "logits/chosen": -2.5403127670288086,
+      "logits/rejected": -2.491334915161133,
+      "logps/chosen": -281.8306884765625,
+      "logps/rejected": -264.59130859375,
+      "loss": 0.6257,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.37535926699638367,
+      "rewards/margins": 0.173495814204216,
+      "rewards/rejected": 0.20186343789100647,
+      "step": 70
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.6033613681793213,
+      "logits/rejected": -2.548762083053589,
+      "logps/chosen": -270.06597900390625,
+      "logps/rejected": -262.29693603515625,
+      "loss": 0.6014,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.4671238958835602,
+      "rewards/margins": 0.144499734044075,
+      "rewards/rejected": 0.3226241171360016,
+      "step": 80
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3437499999999998e-07,
+      "logits/chosen": -2.5922348499298096,
+      "logits/rejected": -2.584137439727783,
+      "logps/chosen": -278.80865478515625,
+      "logps/rejected": -279.4120178222656,
+      "loss": 0.5943,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.295165479183197,
+      "rewards/margins": 0.24514034390449524,
+      "rewards/rejected": 0.05002513527870178,
+      "step": 90
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.604166666666667e-07,
+      "logits/chosen": -2.5200092792510986,
+      "logits/rejected": -2.4520034790039062,
+      "logps/chosen": -240.3234405517578,
+      "logps/rejected": -248.67428588867188,
+      "loss": 0.5838,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.4579353332519531,
+      "rewards/margins": 0.31523841619491577,
+      "rewards/rejected": 0.14269694685935974,
+      "step": 100
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.864583333333333e-07,
+      "logits/chosen": -2.5587222576141357,
+      "logits/rejected": -2.5158984661102295,
+      "logps/chosen": -334.97784423828125,
+      "logps/rejected": -290.39935302734375,
+      "loss": 0.5736,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.39039498567581177,
+      "rewards/margins": 0.49432238936424255,
+      "rewards/rejected": -0.10392741858959198,
+      "step": 110
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.5714714527130127,
+      "logits/rejected": -2.503490924835205,
+      "logps/chosen": -287.0711364746094,
+      "logps/rejected": -279.7371520996094,
+      "loss": 0.5872,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.7893859148025513,
+      "rewards/margins": 0.6542637944221497,
+      "rewards/rejected": 0.13512210547924042,
+      "step": 120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.3854166666666667e-07,
+      "logits/chosen": -2.6581666469573975,
+      "logits/rejected": -2.558701753616333,
+      "logps/chosen": -318.14178466796875,
+      "logps/rejected": -277.52838134765625,
+      "loss": 0.5618,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.3024521768093109,
+      "rewards/margins": 0.6436794400215149,
+      "rewards/rejected": -0.3412272334098816,
+      "step": 130
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.645833333333333e-07,
+      "logits/chosen": -2.6539013385772705,
+      "logits/rejected": -2.6091341972351074,
+      "logps/chosen": -307.194091796875,
+      "logps/rejected": -285.154541015625,
+      "loss": 0.5479,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1616998016834259,
+      "rewards/margins": 0.4194749891757965,
+      "rewards/rejected": -0.5811747908592224,
+      "step": 140
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.9062499999999997e-07,
+      "logits/chosen": -2.4873158931732178,
+      "logits/rejected": -2.504676342010498,
+      "logps/chosen": -300.03448486328125,
+      "logps/rejected": -318.03668212890625,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.03602635860443115,
+      "rewards/margins": -0.06406474858522415,
+      "rewards/rejected": 0.028038373216986656,
+      "step": 150
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.6210286617279053,
+      "logits/rejected": -2.6042556762695312,
+      "logps/chosen": -286.2755432128906,
+      "logps/rejected": -299.8374328613281,
+      "loss": 0.5376,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.1890409290790558,
+      "rewards/margins": 0.8602690696716309,
+      "rewards/rejected": -0.6712281703948975,
+      "step": 160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.427083333333333e-07,
+      "logits/chosen": -2.663649320602417,
+      "logits/rejected": -2.611992835998535,
+      "logps/chosen": -312.26580810546875,
+      "logps/rejected": -271.50897216796875,
+      "loss": 0.5935,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.08057786524295807,
+      "rewards/margins": 0.7988370656967163,
+      "rewards/rejected": -0.8794149160385132,
+      "step": 170
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": -2.6343648433685303,
+      "logits/rejected": -2.5704476833343506,
+      "logps/chosen": -338.59124755859375,
+      "logps/rejected": -286.7225341796875,
+      "loss": 0.5162,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.006776231341063976,
+      "rewards/margins": 0.9450720548629761,
+      "rewards/rejected": -0.9518482089042664,
+      "step": 180
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.947916666666667e-07,
+      "logits/chosen": -2.6279683113098145,
+      "logits/rejected": -2.5350871086120605,
+      "logps/chosen": -288.70184326171875,
+      "logps/rejected": -262.1229248046875,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.2994307279586792,
+      "rewards/margins": 0.9815131425857544,
+      "rewards/rejected": -0.6820824146270752,
+      "step": 190
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.976730657358929e-07,
+      "logits/chosen": -2.5379586219787598,
+      "logits/rejected": -2.49354887008667,
+      "logps/chosen": -253.19580078125,
+      "logps/rejected": -223.9005584716797,
+      "loss": 0.8012,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.20366668701171875,
+      "rewards/margins": -0.2523919641971588,
+      "rewards/rejected": 0.04872531443834305,
+      "step": 200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.947643979057592e-07,
+      "logits/chosen": -2.543163299560547,
+      "logits/rejected": -2.5506227016448975,
+      "logps/chosen": -285.0458984375,
+      "logps/rejected": -312.3334045410156,
+      "loss": 0.5666,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.23225872218608856,
+      "rewards/margins": 0.626153826713562,
+      "rewards/rejected": -0.39389508962631226,
+      "step": 210
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.918557300756254e-07,
+      "logits/chosen": -2.4932289123535156,
+      "logits/rejected": -2.464277744293213,
+      "logps/chosen": -292.6415100097656,
+      "logps/rejected": -273.45648193359375,
+      "loss": 0.5932,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.0871698409318924,
+      "rewards/margins": 0.7229372262954712,
+      "rewards/rejected": -0.6357674598693848,
+      "step": 220
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.889470622454916e-07,
+      "logits/chosen": -2.579195261001587,
+      "logits/rejected": -2.464480400085449,
+      "logps/chosen": -310.99090576171875,
+      "logps/rejected": -287.1470031738281,
+      "loss": 0.5378,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.05043240636587143,
+      "rewards/margins": 0.7342410087585449,
+      "rewards/rejected": -0.6838085651397705,
+      "step": 230
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.860383944153578e-07,
+      "logits/chosen": -2.4707000255584717,
+      "logits/rejected": -2.3995609283447266,
+      "logps/chosen": -292.8463439941406,
+      "logps/rejected": -262.3541259765625,
+      "loss": 0.5575,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.05063791200518608,
+      "rewards/margins": 0.8500161170959473,
+      "rewards/rejected": -0.9006540179252625,
+      "step": 240
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.83129726585224e-07,
+      "logits/chosen": -2.4374887943267822,
+      "logits/rejected": -2.4220387935638428,
+      "logps/chosen": -266.37774658203125,
+      "logps/rejected": -283.33978271484375,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.49987784028053284,
+      "rewards/margins": 0.5630615949630737,
+      "rewards/rejected": -1.0629395246505737,
+      "step": 250
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.802210587550902e-07,
+      "logits/chosen": -2.405151844024658,
+      "logits/rejected": -2.373051643371582,
+      "logps/chosen": -323.4302062988281,
+      "logps/rejected": -306.20367431640625,
+      "loss": 0.7053,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.31794676184654236,
+      "rewards/margins": 0.897225558757782,
+      "rewards/rejected": -1.215172290802002,
+      "step": 260
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.773123909249563e-07,
+      "logits/chosen": -2.432044506072998,
+      "logits/rejected": -2.363980770111084,
+      "logps/chosen": -278.9121398925781,
+      "logps/rejected": -302.1050109863281,
+      "loss": 0.6258,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.047296181321144104,
+      "rewards/margins": 0.663731038570404,
+      "rewards/rejected": -0.711027204990387,
+      "step": 270
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.7440372309482255e-07,
+      "logits/chosen": -2.437777042388916,
+      "logits/rejected": -2.3604378700256348,
+      "logps/chosen": -301.4665222167969,
+      "logps/rejected": -265.2036437988281,
+      "loss": 0.7178,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1992354393005371,
+      "rewards/margins": 0.936697781085968,
+      "rewards/rejected": -1.13593327999115,
+      "step": 280
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.7149505526468876e-07,
+      "logits/chosen": -2.397371530532837,
+      "logits/rejected": -2.3492469787597656,
+      "logps/chosen": -296.2301940917969,
+      "logps/rejected": -297.2955322265625,
+      "loss": 0.5162,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.45999306440353394,
+      "rewards/margins": 0.6742507815361023,
+      "rewards/rejected": -0.21425779163837433,
+      "step": 290
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.6858638743455497e-07,
+      "logits/chosen": -2.4259443283081055,
+      "logits/rejected": -2.3763365745544434,
+      "logps/chosen": -349.0486755371094,
+      "logps/rejected": -336.35894775390625,
+      "loss": 0.5774,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.4317784905433655,
+      "rewards/margins": 0.9294607043266296,
+      "rewards/rejected": -0.49768227338790894,
+      "step": 300
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.656777196044212e-07,
+      "logits/chosen": -2.4149937629699707,
+      "logits/rejected": -2.299372911453247,
+      "logps/chosen": -296.93731689453125,
+      "logps/rejected": -269.88836669921875,
+      "loss": 0.614,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13364040851593018,
+      "rewards/margins": 1.1540381908416748,
+      "rewards/rejected": -1.020397663116455,
+      "step": 310
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.627690517742874e-07,
+      "logits/chosen": -2.3246355056762695,
+      "logits/rejected": -2.31697940826416,
+      "logps/chosen": -246.38571166992188,
+      "logps/rejected": -243.2635955810547,
+      "loss": 0.5455,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.3853219151496887,
+      "rewards/margins": 1.1966570615768433,
+      "rewards/rejected": -0.8113352060317993,
+      "step": 320
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.5986038394415354e-07,
+      "logits/chosen": -2.377342939376831,
+      "logits/rejected": -2.266279697418213,
+      "logps/chosen": -291.89923095703125,
+      "logps/rejected": -252.13156127929688,
+      "loss": 0.7187,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.3700163662433624,
+      "rewards/margins": 1.021303415298462,
+      "rewards/rejected": -1.3913196325302124,
+      "step": 330
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.569517161140198e-07,
+      "logits/chosen": -2.4199347496032715,
+      "logits/rejected": -2.4432175159454346,
+      "logps/chosen": -294.2767333984375,
+      "logps/rejected": -317.6234130859375,
+      "loss": 0.5903,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.04284637048840523,
+      "rewards/margins": 0.6986137628555298,
+      "rewards/rejected": -0.6557673811912537,
+      "step": 340
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.5404304828388595e-07,
+      "logits/chosen": -2.488062858581543,
+      "logits/rejected": -2.420581340789795,
+      "logps/chosen": -316.81109619140625,
+      "logps/rejected": -355.86297607421875,
+      "loss": 0.5148,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.16860897839069366,
+      "rewards/margins": 1.251904010772705,
+      "rewards/rejected": -1.4205129146575928,
+      "step": 350
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.5113438045375216e-07,
+      "logits/chosen": -2.372753858566284,
+      "logits/rejected": -2.3147406578063965,
+      "logps/chosen": -287.305419921875,
+      "logps/rejected": -256.7542724609375,
+      "loss": 0.526,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.243671253323555,
+      "rewards/margins": 0.9904806017875671,
+      "rewards/rejected": -0.7468093633651733,
+      "step": 360
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.4822571262361837e-07,
+      "logits/chosen": -2.446676731109619,
+      "logits/rejected": -2.414977550506592,
+      "logps/chosen": -242.71127319335938,
+      "logps/rejected": -246.08462524414062,
+      "loss": 0.6242,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.48434001207351685,
+      "rewards/margins": 0.3427623510360718,
+      "rewards/rejected": -0.8271023631095886,
+      "step": 370
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.453170447934846e-07,
+      "logits/chosen": -2.3129241466522217,
+      "logits/rejected": -2.2660763263702393,
+      "logps/chosen": -325.49517822265625,
+      "logps/rejected": -257.01812744140625,
+      "loss": 0.6439,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.7354428768157959,
+      "rewards/margins": 0.5178254842758179,
+      "rewards/rejected": -1.2532682418823242,
+      "step": 380
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.424083769633508e-07,
+      "logits/chosen": -2.407226085662842,
+      "logits/rejected": -2.3878862857818604,
+      "logps/chosen": -321.883056640625,
+      "logps/rejected": -329.2529296875,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.6276373863220215,
+      "rewards/margins": 0.8156329989433289,
+      "rewards/rejected": -1.4432705640792847,
+      "step": 390
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.39499709133217e-07,
+      "logits/chosen": -2.5650746822357178,
+      "logits/rejected": -2.5109097957611084,
+      "logps/chosen": -315.965576171875,
+      "logps/rejected": -329.1568298339844,
+      "loss": 0.5373,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.22043700516223907,
+      "rewards/margins": 0.7228590250015259,
+      "rewards/rejected": -0.9432960748672485,
+      "step": 400
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.3659104130308314e-07,
+      "logits/chosen": -2.3766233921051025,
+      "logits/rejected": -2.342545747756958,
+      "logps/chosen": -270.418701171875,
+      "logps/rejected": -277.3287048339844,
+      "loss": 0.4702,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.33475637435913086,
+      "rewards/margins": 1.0519344806671143,
+      "rewards/rejected": -1.3866908550262451,
+      "step": 410
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.336823734729494e-07,
+      "logits/chosen": -2.5363078117370605,
+      "logits/rejected": -2.420921802520752,
+      "logps/chosen": -339.0617370605469,
+      "logps/rejected": -314.2167663574219,
+      "loss": 0.5078,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.20719392597675323,
+      "rewards/margins": 1.039120078086853,
+      "rewards/rejected": -1.2463139295578003,
+      "step": 420
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.3077370564281556e-07,
+      "logits/chosen": -2.4507744312286377,
+      "logits/rejected": -2.3617775440216064,
+      "logps/chosen": -305.3289794921875,
+      "logps/rejected": -270.58843994140625,
+      "loss": 0.5647,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5011049509048462,
+      "rewards/margins": 0.8858562707901001,
+      "rewards/rejected": -1.3869613409042358,
+      "step": 430
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.278650378126818e-07,
+      "logits/chosen": -2.327728033065796,
+      "logits/rejected": -2.2920546531677246,
+      "logps/chosen": -289.3099365234375,
+      "logps/rejected": -333.728271484375,
+      "loss": 0.5112,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.1836535632610321,
+      "rewards/margins": 1.081466794013977,
+      "rewards/rejected": -0.8978131413459778,
+      "step": 440
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.24956369982548e-07,
+      "logits/chosen": -2.4250426292419434,
+      "logits/rejected": -2.3737356662750244,
+      "logps/chosen": -308.11236572265625,
+      "logps/rejected": -295.8037109375,
+      "loss": 0.5505,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.22618088126182556,
+      "rewards/margins": 0.990483283996582,
+      "rewards/rejected": -0.7643024325370789,
+      "step": 450
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.220477021524142e-07,
+      "logits/chosen": -2.4373867511749268,
+      "logits/rejected": -2.400118112564087,
+      "logps/chosen": -290.0523681640625,
+      "logps/rejected": -255.9022674560547,
+      "loss": 0.5815,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.12436334788799286,
+      "rewards/margins": 0.7458918690681458,
+      "rewards/rejected": -0.6215284466743469,
+      "step": 460
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.191390343222804e-07,
+      "logits/chosen": -2.506086826324463,
+      "logits/rejected": -2.4103667736053467,
+      "logps/chosen": -305.6959533691406,
+      "logps/rejected": -305.922607421875,
+      "loss": 0.505,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.27497127652168274,
+      "rewards/margins": 1.060005784034729,
+      "rewards/rejected": -0.7850344181060791,
+      "step": 470
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.162303664921466e-07,
+      "logits/chosen": -2.460691213607788,
+      "logits/rejected": -2.3934152126312256,
+      "logps/chosen": -310.1314697265625,
+      "logps/rejected": -297.3235168457031,
+      "loss": 0.5853,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.14773890376091003,
+      "rewards/margins": 0.7825302481651306,
+      "rewards/rejected": -0.6347913146018982,
+      "step": 480
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.1332169866201275e-07,
+      "logits/chosen": -2.4345953464508057,
+      "logits/rejected": -2.4074747562408447,
+      "logps/chosen": -290.62579345703125,
+      "logps/rejected": -277.2015380859375,
+      "loss": 0.5713,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.5942241549491882,
+      "rewards/margins": 0.8701769113540649,
+      "rewards/rejected": -0.2759527266025543,
+      "step": 490
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.10413030831879e-07,
+      "logits/chosen": -2.491701602935791,
+      "logits/rejected": -2.3788347244262695,
+      "logps/chosen": -327.69097900390625,
+      "logps/rejected": -256.1454162597656,
+      "loss": 0.5509,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.3082984387874603,
+      "rewards/margins": 0.7658659219741821,
+      "rewards/rejected": -0.4575675129890442,
+      "step": 500
+    },
+    {
+      "epoch": 0.26,
+      "eval_logits/chosen": -2.4869401454925537,
+      "eval_logits/rejected": -2.4248592853546143,
+      "eval_logps/chosen": -292.5533447265625,
+      "eval_logps/rejected": -277.5867614746094,
+      "eval_loss": 0.6311370134353638,
+      "eval_rewards/accuracies": 0.761904776096344,
+      "eval_rewards/chosen": 0.8113744854927063,
+      "eval_rewards/margins": 0.9294369220733643,
+      "eval_rewards/rejected": -0.11806251108646393,
+      "eval_runtime": 615.8679,
+      "eval_samples_per_second": 3.247,
+      "eval_steps_per_second": 0.102,
+      "step": 500
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.0750436300174517e-07,
+      "logits/chosen": -2.397420883178711,
+      "logits/rejected": -2.372736692428589,
+      "logps/chosen": -287.88897705078125,
+      "logps/rejected": -266.78594970703125,
+      "loss": 0.5289,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.6826907396316528,
+      "rewards/margins": 1.225263237953186,
+      "rewards/rejected": -0.542572557926178,
+      "step": 510
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.0459569517161143e-07,
+      "logits/chosen": -2.448612928390503,
+      "logits/rejected": -2.335453748703003,
+      "logps/chosen": -289.3345947265625,
+      "logps/rejected": -280.9012145996094,
+      "loss": 0.9872,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.408169686794281,
+      "rewards/margins": 1.0372252464294434,
+      "rewards/rejected": -0.6290556192398071,
+      "step": 520
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.016870273414776e-07,
+      "logits/chosen": -2.4662892818450928,
+      "logits/rejected": -2.425401449203491,
+      "logps/chosen": -260.6138000488281,
+      "logps/rejected": -284.0682373046875,
+      "loss": 0.5718,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.2825961709022522,
+      "rewards/margins": 1.0750539302825928,
+      "rewards/rejected": -0.7924576997756958,
+      "step": 530
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 3.987783595113438e-07,
+      "logits/chosen": -2.498586416244507,
+      "logits/rejected": -2.421823501586914,
+      "logps/chosen": -339.8152160644531,
+      "logps/rejected": -266.1904602050781,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.5165579915046692,
+      "rewards/margins": 1.7178938388824463,
+      "rewards/rejected": -1.2013359069824219,
+      "step": 540
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.9586969168121e-07,
+      "logits/chosen": -2.491562604904175,
+      "logits/rejected": -2.4570322036743164,
+      "logps/chosen": -348.4375,
+      "logps/rejected": -306.64697265625,
+      "loss": 0.5497,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.10518566519021988,
+      "rewards/margins": 0.8593299984931946,
+      "rewards/rejected": -0.7541443705558777,
+      "step": 550
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.929610238510762e-07,
+      "logits/chosen": -2.3990979194641113,
+      "logits/rejected": -2.39212965965271,
+      "logps/chosen": -289.9490661621094,
+      "logps/rejected": -269.9928894042969,
+      "loss": 0.6319,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04818814992904663,
+      "rewards/margins": 0.6179074048995972,
+      "rewards/rejected": -0.569719135761261,
+      "step": 560
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 3.900523560209424e-07,
+      "logits/chosen": -2.4815762042999268,
+      "logits/rejected": -2.425726890563965,
+      "logps/chosen": -312.6693420410156,
+      "logps/rejected": -273.46728515625,
+      "loss": 0.5107,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.3141094148159027,
+      "rewards/margins": 1.199885606765747,
+      "rewards/rejected": -0.8857762217521667,
+      "step": 570
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 3.871436881908086e-07,
+      "logits/chosen": -2.5039753913879395,
+      "logits/rejected": -2.414092779159546,
+      "logps/chosen": -284.1502990722656,
+      "logps/rejected": -240.26083374023438,
+      "loss": 0.523,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.05462036281824112,
+      "rewards/margins": 0.9068329930305481,
+      "rewards/rejected": -0.8522126078605652,
+      "step": 580
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.842350203606748e-07,
+      "logits/chosen": -2.487733840942383,
+      "logits/rejected": -2.4456124305725098,
+      "logps/chosen": -264.53790283203125,
+      "logps/rejected": -287.30718994140625,
+      "loss": 0.5847,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.48793086409568787,
+      "rewards/margins": 1.0478885173797607,
+      "rewards/rejected": -1.535819411277771,
+      "step": 590
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.8132635253054103e-07,
+      "logits/chosen": -2.4525985717773438,
+      "logits/rejected": -2.419036865234375,
+      "logps/chosen": -292.17840576171875,
+      "logps/rejected": -305.83258056640625,
+      "loss": 0.5466,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03021824359893799,
+      "rewards/margins": 1.0404767990112305,
+      "rewards/rejected": -1.070695161819458,
+      "step": 600
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.784176847004072e-07,
+      "logits/chosen": -2.494528293609619,
+      "logits/rejected": -2.455793857574463,
+      "logps/chosen": -276.6438903808594,
+      "logps/rejected": -284.5893249511719,
+      "loss": 0.4859,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.11220204830169678,
+      "rewards/margins": 1.3604462146759033,
+      "rewards/rejected": -1.2482441663742065,
+      "step": 610
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.755090168702734e-07,
+      "logits/chosen": -2.4499683380126953,
+      "logits/rejected": -2.4269161224365234,
+      "logps/chosen": -319.4291076660156,
+      "logps/rejected": -330.466552734375,
+      "loss": 0.5209,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1047038584947586,
+      "rewards/margins": 1.4205875396728516,
+      "rewards/rejected": -1.5252914428710938,
+      "step": 620
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.726003490401396e-07,
+      "logits/chosen": -2.4178521633148193,
+      "logits/rejected": -2.3652777671813965,
+      "logps/chosen": -301.78399658203125,
+      "logps/rejected": -255.4466094970703,
+      "loss": 0.6127,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.73511803150177,
+      "rewards/margins": 0.9625850915908813,
+      "rewards/rejected": -1.6977031230926514,
+      "step": 630
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.696916812100058e-07,
+      "logits/chosen": -2.414977788925171,
+      "logits/rejected": -2.2954540252685547,
+      "logps/chosen": -309.4778747558594,
+      "logps/rejected": -232.9076385498047,
+      "loss": 0.563,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.36078378558158875,
+      "rewards/margins": 1.096639633178711,
+      "rewards/rejected": -1.457423448562622,
+      "step": 640
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 3.66783013379872e-07,
+      "logits/chosen": -2.4305996894836426,
+      "logits/rejected": -2.3547043800354004,
+      "logps/chosen": -303.08221435546875,
+      "logps/rejected": -249.44003295898438,
+      "loss": 0.5194,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.32232993841171265,
+      "rewards/margins": 1.2769782543182373,
+      "rewards/rejected": -1.5993082523345947,
+      "step": 650
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.6387434554973823e-07,
+      "logits/chosen": -2.4183077812194824,
+      "logits/rejected": -2.3758342266082764,
+      "logps/chosen": -326.63739013671875,
+      "logps/rejected": -319.60357666015625,
+      "loss": 0.5133,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7344697117805481,
+      "rewards/margins": 1.0054800510406494,
+      "rewards/rejected": -1.7399498224258423,
+      "step": 660
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.609656777196044e-07,
+      "logits/chosen": -2.4492297172546387,
+      "logits/rejected": -2.3460309505462646,
+      "logps/chosen": -334.344970703125,
+      "logps/rejected": -272.79498291015625,
+      "loss": 0.5318,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.6380278468132019,
+      "rewards/margins": 1.2398582696914673,
+      "rewards/rejected": -1.877886414527893,
+      "step": 670
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.5805700988947064e-07,
+      "logits/chosen": -2.37317156791687,
+      "logits/rejected": -2.287410259246826,
+      "logps/chosen": -259.41107177734375,
+      "logps/rejected": -246.9707794189453,
+      "loss": 0.6405,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.004200550727546215,
+      "rewards/margins": 1.685315728187561,
+      "rewards/rejected": -1.689516305923462,
+      "step": 680
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.551483420593368e-07,
+      "logits/chosen": -2.424691677093506,
+      "logits/rejected": -2.3706610202789307,
+      "logps/chosen": -337.8548889160156,
+      "logps/rejected": -300.6462707519531,
+      "loss": 0.4984,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.4729352593421936,
+      "rewards/margins": 1.0950307846069336,
+      "rewards/rejected": -1.567966103553772,
+      "step": 690
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.5223967422920306e-07,
+      "logits/chosen": -2.4483962059020996,
+      "logits/rejected": -2.3456740379333496,
+      "logps/chosen": -355.38446044921875,
+      "logps/rejected": -313.3027038574219,
+      "loss": 0.7153,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5108211636543274,
+      "rewards/margins": 1.1368777751922607,
+      "rewards/rejected": -1.647698998451233,
+      "step": 700
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.493310063990692e-07,
+      "logits/chosen": -2.4961283206939697,
+      "logits/rejected": -2.4019272327423096,
+      "logps/chosen": -258.533447265625,
+      "logps/rejected": -235.88912963867188,
+      "loss": 0.6123,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2880368232727051,
+      "rewards/margins": 0.940460205078125,
+      "rewards/rejected": -1.22849702835083,
+      "step": 710
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.464223385689354e-07,
+      "logits/chosen": -2.4469172954559326,
+      "logits/rejected": -2.442150354385376,
+      "logps/chosen": -263.5059814453125,
+      "logps/rejected": -278.46112060546875,
+      "loss": 0.5192,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03521687909960747,
+      "rewards/margins": 1.297978162765503,
+      "rewards/rejected": -1.2627613544464111,
+      "step": 720
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.4351367073880163e-07,
+      "logits/chosen": -2.567368745803833,
+      "logits/rejected": -2.5503649711608887,
+      "logps/chosen": -263.9495544433594,
+      "logps/rejected": -274.8172302246094,
+      "loss": 0.4936,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.18824487924575806,
+      "rewards/margins": 1.1973415613174438,
+      "rewards/rejected": -1.3855865001678467,
+      "step": 730
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.4060500290866783e-07,
+      "logits/chosen": -2.5890707969665527,
+      "logits/rejected": -2.497729539871216,
+      "logps/chosen": -357.546875,
+      "logps/rejected": -316.70361328125,
+      "loss": 0.4672,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.6199100017547607,
+      "rewards/margins": 1.5381364822387695,
+      "rewards/rejected": -2.158046245574951,
+      "step": 740
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.37696335078534e-07,
+      "logits/chosen": -2.6034646034240723,
+      "logits/rejected": -2.5190954208374023,
+      "logps/chosen": -307.06573486328125,
+      "logps/rejected": -264.7110900878906,
+      "loss": 0.5857,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.9259228706359863,
+      "rewards/margins": 1.1956923007965088,
+      "rewards/rejected": -2.121615409851074,
+      "step": 750
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.3478766724840025e-07,
+      "logits/chosen": -2.5817999839782715,
+      "logits/rejected": -2.532043695449829,
+      "logps/chosen": -291.2391662597656,
+      "logps/rejected": -286.722900390625,
+      "loss": 0.4794,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.822766900062561,
+      "rewards/margins": 1.2887961864471436,
+      "rewards/rejected": -2.111563205718994,
+      "step": 760
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.318789994182664e-07,
+      "logits/chosen": -2.596311092376709,
+      "logits/rejected": -2.5423972606658936,
+      "logps/chosen": -271.95330810546875,
+      "logps/rejected": -298.80975341796875,
+      "loss": 0.5103,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.4673972725868225,
+      "rewards/margins": 1.1896220445632935,
+      "rewards/rejected": -1.6570192575454712,
+      "step": 770
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.2897033158813266e-07,
+      "logits/chosen": -2.5218849182128906,
+      "logits/rejected": -2.5217957496643066,
+      "logps/chosen": -309.8044738769531,
+      "logps/rejected": -282.1851806640625,
+      "loss": 0.584,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8506789207458496,
+      "rewards/margins": 0.7243373990058899,
+      "rewards/rejected": -1.5750162601470947,
+      "step": 780
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.260616637579988e-07,
+      "logits/chosen": -2.621936082839966,
+      "logits/rejected": -2.5529212951660156,
+      "logps/chosen": -321.58624267578125,
+      "logps/rejected": -306.6531677246094,
+      "loss": 0.5215,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.5436080694198608,
+      "rewards/margins": 1.283302903175354,
+      "rewards/rejected": -1.8269107341766357,
+      "step": 790
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.2315299592786503e-07,
+      "logits/chosen": -2.537576913833618,
+      "logits/rejected": -2.497835636138916,
+      "logps/chosen": -290.63714599609375,
+      "logps/rejected": -284.85650634765625,
+      "loss": 0.5535,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.6990116834640503,
+      "rewards/margins": 1.495788812637329,
+      "rewards/rejected": -2.194800615310669,
+      "step": 800
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.2024432809773123e-07,
+      "logits/chosen": -2.486382246017456,
+      "logits/rejected": -2.403653860092163,
+      "logps/chosen": -286.6750183105469,
+      "logps/rejected": -267.35723876953125,
+      "loss": 0.5029,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0650520324707031,
+      "rewards/margins": 1.3269193172454834,
+      "rewards/rejected": -2.3919713497161865,
+      "step": 810
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.1733566026759744e-07,
+      "logits/chosen": -2.5125479698181152,
+      "logits/rejected": -2.461850643157959,
+      "logps/chosen": -289.32379150390625,
+      "logps/rejected": -280.98785400390625,
+      "loss": 0.4962,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6875102519989014,
+      "rewards/margins": 1.0723689794540405,
+      "rewards/rejected": -1.7598793506622314,
+      "step": 820
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.144269924374636e-07,
+      "logits/chosen": -2.495903491973877,
+      "logits/rejected": -2.4473767280578613,
+      "logps/chosen": -277.972900390625,
+      "logps/rejected": -305.8498840332031,
+      "loss": 0.5665,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.6580140590667725,
+      "rewards/margins": 0.9823113679885864,
+      "rewards/rejected": -1.6403253078460693,
+      "step": 830
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.1151832460732986e-07,
+      "logits/chosen": -2.4640355110168457,
+      "logits/rejected": -2.3490617275238037,
+      "logps/chosen": -321.24688720703125,
+      "logps/rejected": -300.0312805175781,
+      "loss": 0.7062,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.7375732660293579,
+      "rewards/margins": 1.0734766721725464,
+      "rewards/rejected": -1.8110501766204834,
+      "step": 840
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.08609656777196e-07,
+      "logits/chosen": -2.4589717388153076,
+      "logits/rejected": -2.4391770362854004,
+      "logps/chosen": -276.55450439453125,
+      "logps/rejected": -291.0248107910156,
+      "loss": 0.5562,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.9284421801567078,
+      "rewards/margins": 1.0043559074401855,
+      "rewards/rejected": -1.9327980279922485,
+      "step": 850
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.0570098894706227e-07,
+      "logits/chosen": -2.4961347579956055,
+      "logits/rejected": -2.4345571994781494,
+      "logps/chosen": -305.3291015625,
+      "logps/rejected": -354.2350769042969,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7021960616111755,
+      "rewards/margins": 1.1666209697723389,
+      "rewards/rejected": -1.8688170909881592,
+      "step": 860
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.0279232111692843e-07,
+      "logits/chosen": -2.366995096206665,
+      "logits/rejected": -2.3412668704986572,
+      "logps/chosen": -269.6289978027344,
+      "logps/rejected": -294.22308349609375,
+      "loss": 0.5518,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.0025119720958173275,
+      "rewards/margins": 1.3077478408813477,
+      "rewards/rejected": -1.3102598190307617,
+      "step": 870
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 2.9988365328679463e-07,
+      "logits/chosen": -2.5405781269073486,
+      "logits/rejected": -2.5027382373809814,
+      "logps/chosen": -310.4794006347656,
+      "logps/rejected": -311.45599365234375,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.14495661854743958,
+      "rewards/margins": 1.4426281452178955,
+      "rewards/rejected": -1.5875847339630127,
+      "step": 880
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.9697498545666084e-07,
+      "logits/chosen": -2.406571388244629,
+      "logits/rejected": -2.3264360427856445,
+      "logps/chosen": -347.81866455078125,
+      "logps/rejected": -290.57080078125,
+      "loss": 0.5883,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.660466730594635,
+      "rewards/margins": 1.2641503810882568,
+      "rewards/rejected": -1.924617052078247,
+      "step": 890
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.9406631762652705e-07,
+      "logits/chosen": -2.4709229469299316,
+      "logits/rejected": -2.382078170776367,
+      "logps/chosen": -340.4996032714844,
+      "logps/rejected": -315.2070617675781,
+      "loss": 0.515,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.6704212427139282,
+      "rewards/margins": 1.4002532958984375,
+      "rewards/rejected": -2.0706748962402344,
+      "step": 900
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.9115764979639326e-07,
+      "logits/chosen": -2.506743907928467,
+      "logits/rejected": -2.4725024700164795,
+      "logps/chosen": -340.2807922363281,
+      "logps/rejected": -301.91082763671875,
+      "loss": 0.4824,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.5034831762313843,
+      "rewards/margins": 1.6047446727752686,
+      "rewards/rejected": -2.1082279682159424,
+      "step": 910
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.8824898196625947e-07,
+      "logits/chosen": -2.446906805038452,
+      "logits/rejected": -2.392117977142334,
+      "logps/chosen": -301.62506103515625,
+      "logps/rejected": -307.46258544921875,
+      "loss": 0.4731,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.13553249835968018,
+      "rewards/margins": 1.2748148441314697,
+      "rewards/rejected": -1.4103472232818604,
+      "step": 920
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.853403141361256e-07,
+      "logits/chosen": -2.418687343597412,
+      "logits/rejected": -2.3500733375549316,
+      "logps/chosen": -301.64056396484375,
+      "logps/rejected": -261.1069030761719,
+      "loss": 0.5395,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.5969820022583008,
+      "rewards/margins": 0.8396062850952148,
+      "rewards/rejected": -1.4365884065628052,
+      "step": 930
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.824316463059919e-07,
+      "logits/chosen": -2.4292044639587402,
+      "logits/rejected": -2.3925364017486572,
+      "logps/chosen": -310.83258056640625,
+      "logps/rejected": -304.67486572265625,
+      "loss": 0.484,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4251963198184967,
+      "rewards/margins": 1.4573180675506592,
+      "rewards/rejected": -1.8825145959854126,
+      "step": 940
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.7952297847585803e-07,
+      "logits/chosen": -2.417309284210205,
+      "logits/rejected": -2.3521695137023926,
+      "logps/chosen": -278.1476745605469,
+      "logps/rejected": -281.66656494140625,
+      "loss": 0.5149,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.28509074449539185,
+      "rewards/margins": 1.083017110824585,
+      "rewards/rejected": -1.3681080341339111,
+      "step": 950
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.766143106457243e-07,
+      "logits/chosen": -2.3309614658355713,
+      "logits/rejected": -2.3364787101745605,
+      "logps/chosen": -259.80975341796875,
+      "logps/rejected": -250.3785400390625,
+      "loss": 0.5875,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.18767808377742767,
+      "rewards/margins": 1.1901742219924927,
+      "rewards/rejected": -1.377852201461792,
+      "step": 960
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.7370564281559045e-07,
+      "logits/chosen": -2.5228657722473145,
+      "logits/rejected": -2.431464672088623,
+      "logps/chosen": -346.1999816894531,
+      "logps/rejected": -306.3692932128906,
+      "loss": 0.5132,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.02729633077979088,
+      "rewards/margins": 1.0325729846954346,
+      "rewards/rejected": -1.0598691701889038,
+      "step": 970
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.7079697498545666e-07,
+      "logits/chosen": -2.458400249481201,
+      "logits/rejected": -2.4137959480285645,
+      "logps/chosen": -322.91864013671875,
+      "logps/rejected": -321.85699462890625,
+      "loss": 0.477,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.24051785469055176,
+      "rewards/margins": 1.2779700756072998,
+      "rewards/rejected": -1.5184879302978516,
+      "step": 980
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.6788830715532287e-07,
+      "logits/chosen": -2.5085196495056152,
+      "logits/rejected": -2.441279888153076,
+      "logps/chosen": -266.5360412597656,
+      "logps/rejected": -265.48980712890625,
+      "loss": 0.58,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.10649283230304718,
+      "rewards/margins": 1.1308716535568237,
+      "rewards/rejected": -1.2373645305633545,
+      "step": 990
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.6497963932518907e-07,
+      "logits/chosen": -2.3860020637512207,
+      "logits/rejected": -2.3790476322174072,
+      "logps/chosen": -304.46270751953125,
+      "logps/rejected": -309.70697021484375,
+      "loss": 0.6191,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.43433743715286255,
+      "rewards/margins": 0.8789669275283813,
+      "rewards/rejected": -1.3133043050765991,
+      "step": 1000
+    },
+    {
+      "epoch": 0.52,
+      "eval_logits/chosen": -2.490722894668579,
+      "eval_logits/rejected": -2.427964210510254,
+      "eval_logps/chosen": -300.2858581542969,
+      "eval_logps/rejected": -286.9600524902344,
+      "eval_loss": 0.5983877778053284,
+      "eval_rewards/accuracies": 0.7599206566810608,
+      "eval_rewards/chosen": 0.03812364488840103,
+      "eval_rewards/margins": 1.0935115814208984,
+      "eval_rewards/rejected": -1.0553878545761108,
+      "eval_runtime": 614.9522,
+      "eval_samples_per_second": 3.252,
+      "eval_steps_per_second": 0.102,
+      "step": 1000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6207097149505523e-07,
+      "logits/chosen": -2.511011838912964,
+      "logits/rejected": -2.427821159362793,
+      "logps/chosen": -329.0254821777344,
+      "logps/rejected": -263.37030029296875,
+      "loss": 0.529,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.06932397931814194,
+      "rewards/margins": 1.145455002784729,
+      "rewards/rejected": -1.0761311054229736,
+      "step": 1010
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.591623036649215e-07,
+      "logits/chosen": -2.5254805088043213,
+      "logits/rejected": -2.4318673610687256,
+      "logps/chosen": -304.84466552734375,
+      "logps/rejected": -272.1488952636719,
+      "loss": 0.4663,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.25631338357925415,
+      "rewards/margins": 1.2234864234924316,
+      "rewards/rejected": -1.4797999858856201,
+      "step": 1020
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5625363583478764e-07,
+      "logits/chosen": -2.4320321083068848,
+      "logits/rejected": -2.407522201538086,
+      "logps/chosen": -285.1934814453125,
+      "logps/rejected": -312.1936950683594,
+      "loss": 0.4631,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.12434335052967072,
+      "rewards/margins": 1.4510531425476074,
+      "rewards/rejected": -1.5753967761993408,
+      "step": 1030
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.533449680046539e-07,
+      "logits/chosen": -2.480064630508423,
+      "logits/rejected": -2.439182758331299,
+      "logps/chosen": -309.5289001464844,
+      "logps/rejected": -296.3352355957031,
+      "loss": 0.5437,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.2124924659729004,
+      "rewards/margins": 1.4312546253204346,
+      "rewards/rejected": -1.643747091293335,
+      "step": 1040
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5043630017452006e-07,
+      "logits/chosen": -2.41044020652771,
+      "logits/rejected": -2.3936314582824707,
+      "logps/chosen": -284.94268798828125,
+      "logps/rejected": -283.9359436035156,
+      "loss": 0.5227,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.03999950736761093,
+      "rewards/margins": 1.149482011795044,
+      "rewards/rejected": -1.1094822883605957,
+      "step": 1050
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4752763234438627e-07,
+      "logits/chosen": -2.430999279022217,
+      "logits/rejected": -2.3697829246520996,
+      "logps/chosen": -253.0732421875,
+      "logps/rejected": -247.62130737304688,
+      "loss": 0.4938,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.35475850105285645,
+      "rewards/margins": 1.3529800176620483,
+      "rewards/rejected": -0.9982213973999023,
+      "step": 1060
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4461896451425247e-07,
+      "logits/chosen": -2.4376707077026367,
+      "logits/rejected": -2.4240877628326416,
+      "logps/chosen": -269.42938232421875,
+      "logps/rejected": -270.54644775390625,
+      "loss": 0.4648,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.1738893836736679,
+      "rewards/margins": 1.2686713933944702,
+      "rewards/rejected": -1.4425609111785889,
+      "step": 1070
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.417102966841187e-07,
+      "logits/chosen": -2.458761692047119,
+      "logits/rejected": -2.3944101333618164,
+      "logps/chosen": -271.7908935546875,
+      "logps/rejected": -296.3941345214844,
+      "loss": 0.5304,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.14974147081375122,
+      "rewards/margins": 1.2848742008209229,
+      "rewards/rejected": -1.4346158504486084,
+      "step": 1080
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3880162885398483e-07,
+      "logits/chosen": -2.4826645851135254,
+      "logits/rejected": -2.375396251678467,
+      "logps/chosen": -299.49560546875,
+      "logps/rejected": -252.5529327392578,
+      "loss": 0.532,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.49217844009399414,
+      "rewards/margins": 1.3745396137237549,
+      "rewards/rejected": -1.8667182922363281,
+      "step": 1090
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.3589296102385107e-07,
+      "logits/chosen": -2.499211072921753,
+      "logits/rejected": -2.417027235031128,
+      "logps/chosen": -299.8446044921875,
+      "logps/rejected": -286.61322021484375,
+      "loss": 0.4399,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.06040573865175247,
+      "rewards/margins": 1.4158607721328735,
+      "rewards/rejected": -1.4762665033340454,
+      "step": 1100
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.3298429319371725e-07,
+      "logits/chosen": -2.5318305492401123,
+      "logits/rejected": -2.441438674926758,
+      "logps/chosen": -301.87762451171875,
+      "logps/rejected": -303.8865051269531,
+      "loss": 0.4871,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.34866777062416077,
+      "rewards/margins": 1.076101541519165,
+      "rewards/rejected": -1.4247692823410034,
+      "step": 1110
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.3007562536358346e-07,
+      "logits/chosen": -2.478132486343384,
+      "logits/rejected": -2.420320749282837,
+      "logps/chosen": -304.8691711425781,
+      "logps/rejected": -313.31951904296875,
+      "loss": 0.5539,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.0543731153011322,
+      "rewards/margins": 1.4534409046173096,
+      "rewards/rejected": -1.507813811302185,
+      "step": 1120
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.2716695753344967e-07,
+      "logits/chosen": -2.452592372894287,
+      "logits/rejected": -2.422971487045288,
+      "logps/chosen": -301.95721435546875,
+      "logps/rejected": -311.69268798828125,
+      "loss": 0.5074,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.30631425976753235,
+      "rewards/margins": 1.4355709552764893,
+      "rewards/rejected": -1.7418854236602783,
+      "step": 1130
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.2425828970331587e-07,
+      "logits/chosen": -2.5267157554626465,
+      "logits/rejected": -2.4898681640625,
+      "logps/chosen": -267.6355285644531,
+      "logps/rejected": -277.17376708984375,
+      "loss": 0.5519,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5024998784065247,
+      "rewards/margins": 1.3192273378372192,
+      "rewards/rejected": -1.8217273950576782,
+      "step": 1140
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.2134962187318208e-07,
+      "logits/chosen": -2.405796527862549,
+      "logits/rejected": -2.327157497406006,
+      "logps/chosen": -305.180908203125,
+      "logps/rejected": -281.836669921875,
+      "loss": 0.5686,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9660147428512573,
+      "rewards/margins": 1.525781512260437,
+      "rewards/rejected": -2.4917960166931152,
+      "step": 1150
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1844095404304826e-07,
+      "logits/chosen": -2.4776437282562256,
+      "logits/rejected": -2.430321455001831,
+      "logps/chosen": -330.463623046875,
+      "logps/rejected": -324.64324951171875,
+      "loss": 0.6433,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.0749146938323975,
+      "rewards/margins": 0.4202847480773926,
+      "rewards/rejected": -1.4951995611190796,
+      "step": 1160
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1553228621291447e-07,
+      "logits/chosen": -2.4651365280151367,
+      "logits/rejected": -2.394700527191162,
+      "logps/chosen": -286.38763427734375,
+      "logps/rejected": -294.4740295410156,
+      "loss": 0.5214,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.9769517779350281,
+      "rewards/margins": 0.8809939622879028,
+      "rewards/rejected": -1.857945442199707,
+      "step": 1170
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.1262361838278068e-07,
+      "logits/chosen": -2.4773755073547363,
+      "logits/rejected": -2.4409890174865723,
+      "logps/chosen": -308.72369384765625,
+      "logps/rejected": -298.4217224121094,
+      "loss": 0.4998,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.7563016414642334,
+      "rewards/margins": 1.2209903001785278,
+      "rewards/rejected": -1.9772918224334717,
+      "step": 1180
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.0971495055264688e-07,
+      "logits/chosen": -2.4952621459960938,
+      "logits/rejected": -2.4689629077911377,
+      "logps/chosen": -291.19232177734375,
+      "logps/rejected": -306.8577575683594,
+      "loss": 0.5187,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.24586816132068634,
+      "rewards/margins": 1.1213955879211426,
+      "rewards/rejected": -1.3672637939453125,
+      "step": 1190
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0680628272251307e-07,
+      "logits/chosen": -2.509241819381714,
+      "logits/rejected": -2.4504573345184326,
+      "logps/chosen": -285.4134826660156,
+      "logps/rejected": -277.93658447265625,
+      "loss": 0.5123,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3394593298435211,
+      "rewards/margins": 1.0499706268310547,
+      "rewards/rejected": -1.3894299268722534,
+      "step": 1200
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0389761489237927e-07,
+      "logits/chosen": -2.5037548542022705,
+      "logits/rejected": -2.4983646869659424,
+      "logps/chosen": -298.67108154296875,
+      "logps/rejected": -290.48687744140625,
+      "loss": 0.5539,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.12395872920751572,
+      "rewards/margins": 1.0312166213989258,
+      "rewards/rejected": -1.1551753282546997,
+      "step": 1210
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 2.0098894706224548e-07,
+      "logits/chosen": -2.514279842376709,
+      "logits/rejected": -2.473151683807373,
+      "logps/chosen": -255.2734375,
+      "logps/rejected": -252.98855590820312,
+      "loss": 0.4852,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.47876453399658203,
+      "rewards/margins": 1.0464788675308228,
+      "rewards/rejected": -1.5252434015274048,
+      "step": 1220
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.980802792321117e-07,
+      "logits/chosen": -2.471043825149536,
+      "logits/rejected": -2.437826156616211,
+      "logps/chosen": -336.73712158203125,
+      "logps/rejected": -337.2440490722656,
+      "loss": 0.5463,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.4187423586845398,
+      "rewards/margins": 1.326642394065857,
+      "rewards/rejected": -1.7453848123550415,
+      "step": 1230
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.9517161140197787e-07,
+      "logits/chosen": -2.5959396362304688,
+      "logits/rejected": -2.5564401149749756,
+      "logps/chosen": -309.3214111328125,
+      "logps/rejected": -300.98138427734375,
+      "loss": 0.5264,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3669760823249817,
+      "rewards/margins": 1.1876031160354614,
+      "rewards/rejected": -1.5545791387557983,
+      "step": 1240
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.9226294357184408e-07,
+      "logits/chosen": -2.498528480529785,
+      "logits/rejected": -2.4267897605895996,
+      "logps/chosen": -316.4427185058594,
+      "logps/rejected": -322.12493896484375,
+      "loss": 0.5139,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.0588008277118206,
+      "rewards/margins": 1.484725832939148,
+      "rewards/rejected": -1.5435266494750977,
+      "step": 1250
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.8935427574171028e-07,
+      "logits/chosen": -2.542093515396118,
+      "logits/rejected": -2.4682421684265137,
+      "logps/chosen": -337.3454284667969,
+      "logps/rejected": -299.1844482421875,
+      "loss": 0.5489,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.02595728635787964,
+      "rewards/margins": 1.281512975692749,
+      "rewards/rejected": -1.307470440864563,
+      "step": 1260
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.864456079115765e-07,
+      "logits/chosen": -2.477086067199707,
+      "logits/rejected": -2.457650661468506,
+      "logps/chosen": -268.6798095703125,
+      "logps/rejected": -257.23358154296875,
+      "loss": 0.4939,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.1567353755235672,
+      "rewards/margins": 1.2585152387619019,
+      "rewards/rejected": -1.415250539779663,
+      "step": 1270
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.835369400814427e-07,
+      "logits/chosen": -2.544891595840454,
+      "logits/rejected": -2.4903788566589355,
+      "logps/chosen": -289.84234619140625,
+      "logps/rejected": -263.73297119140625,
+      "loss": 0.4819,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.07888557016849518,
+      "rewards/margins": 1.23806893825531,
+      "rewards/rejected": -1.3169543743133545,
+      "step": 1280
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.8062827225130888e-07,
+      "logits/chosen": -2.4458956718444824,
+      "logits/rejected": -2.36130428314209,
+      "logps/chosen": -296.7621765136719,
+      "logps/rejected": -268.7115173339844,
+      "loss": 0.5193,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.21038198471069336,
+      "rewards/margins": 0.9879549741744995,
+      "rewards/rejected": -1.1983369588851929,
+      "step": 1290
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.777196044211751e-07,
+      "logits/chosen": -2.422639846801758,
+      "logits/rejected": -2.3881583213806152,
+      "logps/chosen": -267.2961730957031,
+      "logps/rejected": -258.95159912109375,
+      "loss": 0.5247,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.0472625195980072,
+      "rewards/margins": 1.1689708232879639,
+      "rewards/rejected": -1.216233491897583,
+      "step": 1300
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.748109365910413e-07,
+      "logits/chosen": -2.4238734245300293,
+      "logits/rejected": -2.3859925270080566,
+      "logps/chosen": -241.4824981689453,
+      "logps/rejected": -256.94635009765625,
+      "loss": 0.471,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.039008866995573044,
+      "rewards/margins": 1.5137543678283691,
+      "rewards/rejected": -1.474745512008667,
+      "step": 1310
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.719022687609075e-07,
+      "logits/chosen": -2.5132765769958496,
+      "logits/rejected": -2.449302911758423,
+      "logps/chosen": -363.523681640625,
+      "logps/rejected": -352.0708923339844,
+      "loss": 0.518,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.5855492353439331,
+      "rewards/margins": 1.2664353847503662,
+      "rewards/rejected": -1.8519846200942993,
+      "step": 1320
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.6899360093077368e-07,
+      "logits/chosen": -2.4553329944610596,
+      "logits/rejected": -2.4112277030944824,
+      "logps/chosen": -280.7616882324219,
+      "logps/rejected": -236.74765014648438,
+      "loss": 0.5061,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.48466506600379944,
+      "rewards/margins": 1.0938060283660889,
+      "rewards/rejected": -1.578471064567566,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.660849331006399e-07,
+      "logits/chosen": -2.47243070602417,
+      "logits/rejected": -2.448956251144409,
+      "logps/chosen": -347.1180114746094,
+      "logps/rejected": -303.9145202636719,
+      "loss": 0.5364,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.2242584526538849,
+      "rewards/margins": 1.3924510478973389,
+      "rewards/rejected": -1.6167097091674805,
+      "step": 1340
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.631762652705061e-07,
+      "logits/chosen": -2.430530071258545,
+      "logits/rejected": -2.4229390621185303,
+      "logps/chosen": -245.01284790039062,
+      "logps/rejected": -268.6424560546875,
+      "loss": 0.5447,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5738204121589661,
+      "rewards/margins": 1.0651990175247192,
+      "rewards/rejected": -1.6390196084976196,
+      "step": 1350
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.602675974403723e-07,
+      "logits/chosen": -2.509038209915161,
+      "logits/rejected": -2.489039897918701,
+      "logps/chosen": -279.32379150390625,
+      "logps/rejected": -294.4464416503906,
+      "loss": 0.5021,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5361760854721069,
+      "rewards/margins": 1.5468828678131104,
+      "rewards/rejected": -2.0830588340759277,
+      "step": 1360
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.573589296102385e-07,
+      "logits/chosen": -2.47121262550354,
+      "logits/rejected": -2.3763961791992188,
+      "logps/chosen": -295.9562072753906,
+      "logps/rejected": -312.29022216796875,
+      "loss": 0.5798,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.7061088681221008,
+      "rewards/margins": 0.6994223594665527,
+      "rewards/rejected": -1.4055311679840088,
+      "step": 1370
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.544502617801047e-07,
+      "logits/chosen": -2.4963533878326416,
+      "logits/rejected": -2.4046132564544678,
+      "logps/chosen": -306.95379638671875,
+      "logps/rejected": -269.7267150878906,
+      "loss": 0.5349,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.14952921867370605,
+      "rewards/margins": 1.085407018661499,
+      "rewards/rejected": -1.2349363565444946,
+      "step": 1380
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.515415939499709e-07,
+      "logits/chosen": -2.3877298831939697,
+      "logits/rejected": -2.3528621196746826,
+      "logps/chosen": -306.18292236328125,
+      "logps/rejected": -281.7127990722656,
+      "loss": 0.4883,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.3554263710975647,
+      "rewards/margins": 1.3783906698226929,
+      "rewards/rejected": -1.7338171005249023,
+      "step": 1390
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.486329261198371e-07,
+      "logits/chosen": -2.4248931407928467,
+      "logits/rejected": -2.3428282737731934,
+      "logps/chosen": -265.8216247558594,
+      "logps/rejected": -254.1871795654297,
+      "loss": 0.5616,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4705156683921814,
+      "rewards/margins": 1.1333085298538208,
+      "rewards/rejected": -1.6038239002227783,
+      "step": 1400
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.4572425828970332e-07,
+      "logits/chosen": -2.5125279426574707,
+      "logits/rejected": -2.4069325923919678,
+      "logps/chosen": -317.7018127441406,
+      "logps/rejected": -303.3476867675781,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.4776553511619568,
+      "rewards/margins": 1.0890066623687744,
+      "rewards/rejected": -1.566662073135376,
+      "step": 1410
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.428155904595695e-07,
+      "logits/chosen": -2.402710437774658,
+      "logits/rejected": -2.35219407081604,
+      "logps/chosen": -308.2895202636719,
+      "logps/rejected": -298.3793640136719,
+      "loss": 0.5636,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.43602848052978516,
+      "rewards/margins": 1.1804325580596924,
+      "rewards/rejected": -1.6164608001708984,
+      "step": 1420
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.399069226294357e-07,
+      "logits/chosen": -2.325125217437744,
+      "logits/rejected": -2.265918254852295,
+      "logps/chosen": -255.18252563476562,
+      "logps/rejected": -242.65451049804688,
+      "loss": 0.5112,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.38456660509109497,
+      "rewards/margins": 1.3007227182388306,
+      "rewards/rejected": -1.6852893829345703,
+      "step": 1430
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.3699825479930191e-07,
+      "logits/chosen": -2.444197177886963,
+      "logits/rejected": -2.3448662757873535,
+      "logps/chosen": -335.58575439453125,
+      "logps/rejected": -257.6363525390625,
+      "loss": 0.4915,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1254938691854477,
+      "rewards/margins": 1.4525253772735596,
+      "rewards/rejected": -1.578019380569458,
+      "step": 1440
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 1.3408958696916812e-07,
+      "logits/chosen": -2.432386875152588,
+      "logits/rejected": -2.3468270301818848,
+      "logps/chosen": -264.3630065917969,
+      "logps/rejected": -253.1586456298828,
+      "loss": 0.4865,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.010800900869071484,
+      "rewards/margins": 1.3583967685699463,
+      "rewards/rejected": -1.3691976070404053,
+      "step": 1450
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 1.311809191390343e-07,
+      "logits/chosen": -2.3925037384033203,
+      "logits/rejected": -2.3687844276428223,
+      "logps/chosen": -298.83160400390625,
+      "logps/rejected": -321.2250671386719,
+      "loss": 0.5522,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.13408958911895752,
+      "rewards/margins": 1.2063826322555542,
+      "rewards/rejected": -1.3404719829559326,
+      "step": 1460
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.282722513089005e-07,
+      "logits/chosen": -2.5323238372802734,
+      "logits/rejected": -2.450490713119507,
+      "logps/chosen": -303.0655517578125,
+      "logps/rejected": -273.21636962890625,
+      "loss": 0.4542,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07529834657907486,
+      "rewards/margins": 1.215362787246704,
+      "rewards/rejected": -1.290661096572876,
+      "step": 1470
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.2536358347876672e-07,
+      "logits/chosen": -2.4639523029327393,
+      "logits/rejected": -2.451842784881592,
+      "logps/chosen": -273.89410400390625,
+      "logps/rejected": -288.24395751953125,
+      "loss": 0.5013,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1511283814907074,
+      "rewards/margins": 1.046019196510315,
+      "rewards/rejected": -1.1971476078033447,
+      "step": 1480
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 1.2245491564863293e-07,
+      "logits/chosen": -2.5214879512786865,
+      "logits/rejected": -2.3725059032440186,
+      "logps/chosen": -365.9342041015625,
+      "logps/rejected": -288.50860595703125,
+      "loss": 0.4655,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.24411487579345703,
+      "rewards/margins": 1.6229133605957031,
+      "rewards/rejected": -1.3787983655929565,
+      "step": 1490
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 1.195462478184991e-07,
+      "logits/chosen": -2.4407520294189453,
+      "logits/rejected": -2.403568744659424,
+      "logps/chosen": -266.7708740234375,
+      "logps/rejected": -256.9794006347656,
+      "loss": 0.5342,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.008794727735221386,
+      "rewards/margins": 1.0573989152908325,
+      "rewards/rejected": -1.0486040115356445,
+      "step": 1500
+    },
+    {
+      "epoch": 0.78,
+      "eval_logits/chosen": -2.479328155517578,
+      "eval_logits/rejected": -2.420834541320801,
+      "eval_logps/chosen": -299.6731872558594,
+      "eval_logps/rejected": -287.96630859375,
+      "eval_loss": 0.5819299817085266,
+      "eval_rewards/accuracies": 0.7559523582458496,
+      "eval_rewards/chosen": 0.09938977658748627,
+      "eval_rewards/margins": 1.2554062604904175,
+      "eval_rewards/rejected": -1.1560163497924805,
+      "eval_runtime": 614.7519,
+      "eval_samples_per_second": 3.253,
+      "eval_steps_per_second": 0.102,
+      "step": 1500
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 1.1663757998836531e-07,
+      "logits/chosen": -2.4385485649108887,
+      "logits/rejected": -2.414762496948242,
+      "logps/chosen": -282.46331787109375,
+      "logps/rejected": -271.0342712402344,
+      "loss": 0.5393,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.030192693695425987,
+      "rewards/margins": 1.1163215637207031,
+      "rewards/rejected": -1.0861289501190186,
+      "step": 1510
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.1372891215823152e-07,
+      "logits/chosen": -2.475036144256592,
+      "logits/rejected": -2.431018352508545,
+      "logps/chosen": -273.4059753417969,
+      "logps/rejected": -279.97149658203125,
+      "loss": 0.4813,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.07051552832126617,
+      "rewards/margins": 1.225347638130188,
+      "rewards/rejected": -1.295863151550293,
+      "step": 1520
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.1082024432809772e-07,
+      "logits/chosen": -2.434101104736328,
+      "logits/rejected": -2.3855373859405518,
+      "logps/chosen": -270.7154235839844,
+      "logps/rejected": -257.188720703125,
+      "loss": 0.4726,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.03814787417650223,
+      "rewards/margins": 1.2096832990646362,
+      "rewards/rejected": -1.2478312253952026,
+      "step": 1530
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.0791157649796392e-07,
+      "logits/chosen": -2.509958267211914,
+      "logits/rejected": -2.5057830810546875,
+      "logps/chosen": -268.060791015625,
+      "logps/rejected": -294.9040832519531,
+      "loss": 0.6052,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.09406580030918121,
+      "rewards/margins": 0.9256828427314758,
+      "rewards/rejected": -1.019748568534851,
+      "step": 1540
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.0500290866783013e-07,
+      "logits/chosen": -2.391065835952759,
+      "logits/rejected": -2.3701300621032715,
+      "logps/chosen": -280.59686279296875,
+      "logps/rejected": -243.96798706054688,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.04815078154206276,
+      "rewards/margins": 0.9994159936904907,
+      "rewards/rejected": -0.9512651562690735,
+      "step": 1550
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.0209424083769633e-07,
+      "logits/chosen": -2.428290367126465,
+      "logits/rejected": -2.394292116165161,
+      "logps/chosen": -284.85968017578125,
+      "logps/rejected": -287.145263671875,
+      "loss": 0.505,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.22867484390735626,
+      "rewards/margins": 1.033279299736023,
+      "rewards/rejected": -1.2619540691375732,
+      "step": 1560
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.918557300756253e-08,
+      "logits/chosen": -2.4378373622894287,
+      "logits/rejected": -2.369236946105957,
+      "logps/chosen": -293.5215759277344,
+      "logps/rejected": -246.56997680664062,
+      "loss": 0.4905,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.13024893403053284,
+      "rewards/margins": 1.4804332256317139,
+      "rewards/rejected": -1.6106821298599243,
+      "step": 1570
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.627690517742873e-08,
+      "logits/chosen": -2.4206690788269043,
+      "logits/rejected": -2.409813165664673,
+      "logps/chosen": -333.97467041015625,
+      "logps/rejected": -314.40960693359375,
+      "loss": 0.5211,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.2985494136810303,
+      "rewards/margins": 1.2202610969543457,
+      "rewards/rejected": -0.9217117428779602,
+      "step": 1580
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.336823734729494e-08,
+      "logits/chosen": -2.426513671875,
+      "logits/rejected": -2.4083962440490723,
+      "logps/chosen": -312.02655029296875,
+      "logps/rejected": -314.4944152832031,
+      "loss": 0.5172,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1636694371700287,
+      "rewards/margins": 1.20877206325531,
+      "rewards/rejected": -1.372441291809082,
+      "step": 1590
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 9.045956951716113e-08,
+      "logits/chosen": -2.4410054683685303,
+      "logits/rejected": -2.421079158782959,
+      "logps/chosen": -268.4046630859375,
+      "logps/rejected": -259.6822509765625,
+      "loss": 0.5011,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.018908673897385597,
+      "rewards/margins": 1.4655755758285522,
+      "rewards/rejected": -1.446666955947876,
+      "step": 1600
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.755090168702734e-08,
+      "logits/chosen": -2.4191842079162598,
+      "logits/rejected": -2.3796703815460205,
+      "logps/chosen": -282.9931945800781,
+      "logps/rejected": -297.3874206542969,
+      "loss": 0.488,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.26676809787750244,
+      "rewards/margins": 1.4654661417007446,
+      "rewards/rejected": -1.732234239578247,
+      "step": 1610
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 8.464223385689353e-08,
+      "logits/chosen": -2.474801540374756,
+      "logits/rejected": -2.439415693283081,
+      "logps/chosen": -316.919921875,
+      "logps/rejected": -286.6101989746094,
+      "loss": 0.5092,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.14554055035114288,
+      "rewards/margins": 1.1523851156234741,
+      "rewards/rejected": -1.2979257106781006,
+      "step": 1620
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 8.173356602675974e-08,
+      "logits/chosen": -2.501138210296631,
+      "logits/rejected": -2.423609495162964,
+      "logps/chosen": -267.6105651855469,
+      "logps/rejected": -268.2532653808594,
+      "loss": 0.5253,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7160999178886414,
+      "rewards/margins": 0.7922013998031616,
+      "rewards/rejected": -1.5083013772964478,
+      "step": 1630
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.882489819662593e-08,
+      "logits/chosen": -2.4721310138702393,
+      "logits/rejected": -2.4814186096191406,
+      "logps/chosen": -304.85931396484375,
+      "logps/rejected": -421.44427490234375,
+      "loss": 0.4832,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4838688373565674,
+      "rewards/margins": 1.3697353601455688,
+      "rewards/rejected": -1.8536040782928467,
+      "step": 1640
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.591623036649214e-08,
+      "logits/chosen": -2.331878185272217,
+      "logits/rejected": -2.3361971378326416,
+      "logps/chosen": -296.43096923828125,
+      "logps/rejected": -293.191650390625,
+      "loss": 0.5881,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.4644225537776947,
+      "rewards/margins": 1.1996773481369019,
+      "rewards/rejected": -1.664099931716919,
+      "step": 1650
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 7.300756253635834e-08,
+      "logits/chosen": -2.472486972808838,
+      "logits/rejected": -2.420685291290283,
+      "logps/chosen": -268.98712158203125,
+      "logps/rejected": -287.7394714355469,
+      "loss": 0.4996,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.22886808216571808,
+      "rewards/margins": 1.0310173034667969,
+      "rewards/rejected": -1.259885311126709,
+      "step": 1660
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 7.009889470622454e-08,
+      "logits/chosen": -2.480839967727661,
+      "logits/rejected": -2.405022144317627,
+      "logps/chosen": -327.4227600097656,
+      "logps/rejected": -299.2853088378906,
+      "loss": 0.4532,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.21133089065551758,
+      "rewards/margins": 1.356836199760437,
+      "rewards/rejected": -1.568166971206665,
+      "step": 1670
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.719022687609075e-08,
+      "logits/chosen": -2.4610321521759033,
+      "logits/rejected": -2.3211312294006348,
+      "logps/chosen": -367.6954650878906,
+      "logps/rejected": -312.0871887207031,
+      "loss": 0.4964,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6737295389175415,
+      "rewards/margins": 1.504465103149414,
+      "rewards/rejected": -2.178194522857666,
+      "step": 1680
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.428155904595695e-08,
+      "logits/chosen": -2.350032091140747,
+      "logits/rejected": -2.3311705589294434,
+      "logps/chosen": -309.73712158203125,
+      "logps/rejected": -270.7311096191406,
+      "loss": 0.4952,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.18244963884353638,
+      "rewards/margins": 1.2423228025436401,
+      "rewards/rejected": -1.4247725009918213,
+      "step": 1690
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.137289121582315e-08,
+      "logits/chosen": -2.5109975337982178,
+      "logits/rejected": -2.3817737102508545,
+      "logps/chosen": -335.9524841308594,
+      "logps/rejected": -272.06829833984375,
+      "loss": 0.4946,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.4245755672454834,
+      "rewards/margins": 1.40361487865448,
+      "rewards/rejected": -1.828190565109253,
+      "step": 1700
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.846422338568935e-08,
+      "logits/chosen": -2.4695639610290527,
+      "logits/rejected": -2.414325714111328,
+      "logps/chosen": -304.23468017578125,
+      "logps/rejected": -285.8380432128906,
+      "loss": 0.4495,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3581593334674835,
+      "rewards/margins": 1.1380908489227295,
+      "rewards/rejected": -1.4962502717971802,
+      "step": 1710
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.555555555555555e-08,
+      "logits/chosen": -2.439877986907959,
+      "logits/rejected": -2.3664920330047607,
+      "logps/chosen": -350.97137451171875,
+      "logps/rejected": -327.41302490234375,
+      "loss": 0.5355,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5186697244644165,
+      "rewards/margins": 1.1480152606964111,
+      "rewards/rejected": -1.6666851043701172,
+      "step": 1720
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 5.264688772542175e-08,
+      "logits/chosen": -2.455570697784424,
+      "logits/rejected": -2.409554958343506,
+      "logps/chosen": -292.89715576171875,
+      "logps/rejected": -287.56573486328125,
+      "loss": 0.4603,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.08683060109615326,
+      "rewards/margins": 0.9011920690536499,
+      "rewards/rejected": -0.98802250623703,
+      "step": 1730
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.973821989528795e-08,
+      "logits/chosen": -2.502526044845581,
+      "logits/rejected": -2.46052885055542,
+      "logps/chosen": -266.27947998046875,
+      "logps/rejected": -262.73822021484375,
+      "loss": 0.5074,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.008827775716781616,
+      "rewards/margins": 1.2833327054977417,
+      "rewards/rejected": -1.2921605110168457,
+      "step": 1740
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.682955206515416e-08,
+      "logits/chosen": -2.494965076446533,
+      "logits/rejected": -2.4605846405029297,
+      "logps/chosen": -316.22882080078125,
+      "logps/rejected": -368.8150939941406,
+      "loss": 0.5432,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.25845271348953247,
+      "rewards/margins": 1.4049198627471924,
+      "rewards/rejected": -1.6633726358413696,
+      "step": 1750
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.392088423502036e-08,
+      "logits/chosen": -2.481264352798462,
+      "logits/rejected": -2.5107004642486572,
+      "logps/chosen": -274.2122802734375,
+      "logps/rejected": -280.37933349609375,
+      "loss": 0.5557,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5907906293869019,
+      "rewards/margins": 1.1111408472061157,
+      "rewards/rejected": -1.701931357383728,
+      "step": 1760
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 4.101221640488656e-08,
+      "logits/chosen": -2.4728500843048096,
+      "logits/rejected": -2.408026695251465,
+      "logps/chosen": -308.61956787109375,
+      "logps/rejected": -288.31170654296875,
+      "loss": 0.4805,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.6883036494255066,
+      "rewards/margins": 1.1805965900421143,
+      "rewards/rejected": -1.8689002990722656,
+      "step": 1770
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.810354857475276e-08,
+      "logits/chosen": -2.3637709617614746,
+      "logits/rejected": -2.3329529762268066,
+      "logps/chosen": -324.7041320800781,
+      "logps/rejected": -270.0013122558594,
+      "loss": 0.5471,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.544477105140686,
+      "rewards/margins": 1.064997673034668,
+      "rewards/rejected": -1.6094748973846436,
+      "step": 1780
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.519488074461896e-08,
+      "logits/chosen": -2.447537660598755,
+      "logits/rejected": -2.386204242706299,
+      "logps/chosen": -268.69378662109375,
+      "logps/rejected": -276.32843017578125,
+      "loss": 0.3955,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3514997661113739,
+      "rewards/margins": 1.253027319908142,
+      "rewards/rejected": -1.6045271158218384,
+      "step": 1790
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.228621291448516e-08,
+      "logits/chosen": -2.504725694656372,
+      "logits/rejected": -2.4065499305725098,
+      "logps/chosen": -324.8893127441406,
+      "logps/rejected": -262.2546081542969,
+      "loss": 0.4898,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.5291447043418884,
+      "rewards/margins": 1.2341575622558594,
+      "rewards/rejected": -1.763302206993103,
+      "step": 1800
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.9377545084351366e-08,
+      "logits/chosen": -2.4359753131866455,
+      "logits/rejected": -2.3973264694213867,
+      "logps/chosen": -380.8512878417969,
+      "logps/rejected": -345.4535827636719,
+      "loss": 0.4781,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.39544662833213806,
+      "rewards/margins": 1.4492541551589966,
+      "rewards/rejected": -1.844700813293457,
+      "step": 1810
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6468877254217567e-08,
+      "logits/chosen": -2.455008029937744,
+      "logits/rejected": -2.406989574432373,
+      "logps/chosen": -321.1400451660156,
+      "logps/rejected": -326.90203857421875,
+      "loss": 0.5332,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.45269957184791565,
+      "rewards/margins": 1.45340895652771,
+      "rewards/rejected": -1.9061084985733032,
+      "step": 1820
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.3560209424083768e-08,
+      "logits/chosen": -2.451988458633423,
+      "logits/rejected": -2.390340805053711,
+      "logps/chosen": -319.3875427246094,
+      "logps/rejected": -321.97186279296875,
+      "loss": 0.5775,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.6661805510520935,
+      "rewards/margins": 0.7991847395896912,
+      "rewards/rejected": -1.4653651714324951,
+      "step": 1830
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.065154159394997e-08,
+      "logits/chosen": -2.439758539199829,
+      "logits/rejected": -2.419706106185913,
+      "logps/chosen": -330.24810791015625,
+      "logps/rejected": -308.7535095214844,
+      "loss": 0.5889,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5779649019241333,
+      "rewards/margins": 0.9780583381652832,
+      "rewards/rejected": -1.5560232400894165,
+      "step": 1840
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7742873763816174e-08,
+      "logits/chosen": -2.4510066509246826,
+      "logits/rejected": -2.406909704208374,
+      "logps/chosen": -338.01861572265625,
+      "logps/rejected": -291.27923583984375,
+      "loss": 0.5524,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.372661292552948,
+      "rewards/margins": 1.7962923049926758,
+      "rewards/rejected": -2.1689534187316895,
+      "step": 1850
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4834205933682373e-08,
+      "logits/chosen": -2.5106232166290283,
+      "logits/rejected": -2.4458587169647217,
+      "logps/chosen": -300.896728515625,
+      "logps/rejected": -278.838623046875,
+      "loss": 0.5813,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.4824046492576599,
+      "rewards/margins": 1.491176962852478,
+      "rewards/rejected": -1.9735815525054932,
+      "step": 1860
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1925538103548575e-08,
+      "logits/chosen": -2.4239859580993652,
+      "logits/rejected": -2.373213768005371,
+      "logps/chosen": -280.8854675292969,
+      "logps/rejected": -301.01092529296875,
+      "loss": 0.5219,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.43918657302856445,
+      "rewards/margins": 1.376401662826538,
+      "rewards/rejected": -1.8155882358551025,
+      "step": 1870
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.016870273414776e-09,
+      "logits/chosen": -2.4339070320129395,
+      "logits/rejected": -2.3967947959899902,
+      "logps/chosen": -354.99462890625,
+      "logps/rejected": -309.2855529785156,
+      "loss": 0.4764,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.24851647019386292,
+      "rewards/margins": 1.5067845582962036,
+      "rewards/rejected": -1.7553011178970337,
+      "step": 1880
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.1082024432809765e-09,
+      "logits/chosen": -2.4029338359832764,
+      "logits/rejected": -2.4133036136627197,
+      "logps/chosen": -268.3423767089844,
+      "logps/rejected": -301.10321044921875,
+      "loss": 0.5074,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.8797990083694458,
+      "rewards/margins": 0.8209834098815918,
+      "rewards/rejected": -1.7007821798324585,
+      "step": 1890
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.1995346131471783e-09,
+      "logits/chosen": -2.4465489387512207,
+      "logits/rejected": -2.4295449256896973,
+      "logps/chosen": -268.50006103515625,
+      "logps/rejected": -266.83685302734375,
+      "loss": 0.479,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.589029848575592,
+      "rewards/margins": 1.1926295757293701,
+      "rewards/rejected": -1.781659483909607,
+      "step": 1900
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.9086678301337986e-10,
+      "logits/chosen": -2.482949733734131,
+      "logits/rejected": -2.426959991455078,
+      "logps/chosen": -303.0064392089844,
+      "logps/rejected": -287.6535339355469,
+      "loss": 0.5333,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.44241657853126526,
+      "rewards/margins": 0.9881950616836548,
+      "rewards/rejected": -1.4306116104125977,
+      "step": 1910
+    },
+    {
+      "epoch": 1.0,
+      "step": 1911,
+      "total_flos": 0.0,
+      "train_loss": 0.5469513680846325,
+      "train_runtime": 31987.9403,
+      "train_samples_per_second": 1.911,
+      "train_steps_per_second": 0.06
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1911,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c88deef81a32d1c3f94af5c783e8a1d9ccd20a2714528a73414ef6c3c7ef5361
+size 5371