TX

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-Instruct-v0.2
+tags:
+- alignment_handbook-handbook
+- generated_from_trainer
+datasets:
+- princeton-nlp/mistral-instruct-ultrafeedback
+model-index:
+- name: Mistral-7B-Instruct-v0.2-MI-5e-7
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/tengxiao01/huggingface/runs/o7itmh9u)
+# Mistral-7B-Instruct-v0.2-MI-5e-7
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on the princeton-nlp/mistral-instruct-ultrafeedback dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.5221
+- Rewards/chosen: -0.5602
+- Rewards/rejected: -0.6163
+- Rewards/accuracies: 0.5798
+- Rewards/margins: 0.0561
+- Logps/rejected: -0.6163
+- Logps/chosen: -0.5602
+- Logits/rejected: -3.4633
+- Logits/chosen: -3.4685
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 1.4908        | 0.8573 | 400  | 1.5221          | -0.5602        | -0.6163          | 0.5798             | 0.0561          | -0.6163        | -0.5602      | -3.4633         | -3.4685       |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 0.9987943737441393,
+    "eval_logits/chosen": -3.4656760692596436,
+    "eval_logits/rejected": -3.4604122638702393,
+    "eval_logps/chosen": -0.5584612488746643,
+    "eval_logps/rejected": -0.6149689555168152,
+    "eval_loss": 1.5217351913452148,
+    "eval_rewards/accuracies": 0.5811170339584351,
+    "eval_rewards/chosen": -0.5584612488746643,
+    "eval_rewards/margins": 0.05650780722498894,
+    "eval_rewards/rejected": -0.6149689555168152,
+    "eval_runtime": 435.8552,
+    "eval_samples": 2994,
+    "eval_samples_per_second": 6.869,
+    "eval_steps_per_second": 0.431,
+    "total_flos": 0.0,
+    "train_loss": 1.5188358733070766,
+    "train_runtime": 19352.2078,
+    "train_samples": 59720,
+    "train_samples_per_second": 3.086,
+    "train_steps_per_second": 0.024
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 0.9987943737441393,
+    "eval_logits/chosen": -3.4656760692596436,
+    "eval_logits/rejected": -3.4604122638702393,
+    "eval_logps/chosen": -0.5584612488746643,
+    "eval_logps/rejected": -0.6149689555168152,
+    "eval_loss": 1.5217351913452148,
+    "eval_rewards/accuracies": 0.5811170339584351,
+    "eval_rewards/chosen": -0.5584612488746643,
+    "eval_rewards/margins": 0.05650780722498894,
+    "eval_rewards/rejected": -0.6149689555168152,
+    "eval_runtime": 435.8552,
+    "eval_samples": 2994,
+    "eval_samples_per_second": 6.869,
+    "eval_steps_per_second": 0.431
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.42.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:107c6e7893b9093bdda9c59a2d2b1b61511c2b85e5bb36f884052f1e55c266dd
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18b9dc318b9b7acaa5648eef19a15e18fc7fe86f5517fbb614160726818204ed
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:340ff1ba711f154010cf81f7f6185f18fa8ec210ba4657215248f1f7d5290706
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n        {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif %}\n    {%- if message['role'] == 'user' %}\n        {%- if loop.first and system_message is defined %}\n            {{- ' [INST] ' + system_message + '\\n\\n' + message['content'] + ' [/INST]' }}\n        {%- else %}\n            {{- ' [INST] ' + message['content'] + ' [/INST]' }}\n        {%- endif %}\n    {%- elif message['role'] == 'assistant' %}\n        {{- ' ' + message['content'] + eos_token}}\n    {%- else %}\n        {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n    {%- endif %}\n{%- endfor %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9987943737441393,
+    "total_flos": 0.0,
+    "train_loss": 1.5188358733070766,
+    "train_runtime": 19352.2078,
+    "train_samples": 59720,
+    "train_samples_per_second": 3.086,
+    "train_steps_per_second": 0.024
+}

trainer_state.json

@@ -0,0 +1,1453 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9987943737441393,
+  "eval_steps": 400,
+  "global_step": 466,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010716677829872739,
+      "grad_norm": 38.485095124933046,
+      "learning_rate": 5.3191489361702123e-08,
+      "logits/chosen": -2.398301601409912,
+      "logits/rejected": -2.3920369148254395,
+      "logps/chosen": -0.5667670965194702,
+      "logps/rejected": -0.5555010437965393,
+      "loss": 1.5468,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.5667670965194702,
+      "rewards/margins": -0.011266030371189117,
+      "rewards/rejected": -0.5555010437965393,
+      "step": 5
+    },
+    {
+      "epoch": 0.021433355659745478,
+      "grad_norm": 18.322790262213086,
+      "learning_rate": 1.0638297872340425e-07,
+      "logits/chosen": -2.4026951789855957,
+      "logits/rejected": -2.3731987476348877,
+      "logps/chosen": -0.5517200231552124,
+      "logps/rejected": -0.5781593918800354,
+      "loss": 1.554,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.5517200231552124,
+      "rewards/margins": 0.026439335197210312,
+      "rewards/rejected": -0.5781593918800354,
+      "step": 10
+    },
+    {
+      "epoch": 0.032150033489618215,
+      "grad_norm": 17.062192600734857,
+      "learning_rate": 1.5957446808510638e-07,
+      "logits/chosen": -2.4439353942871094,
+      "logits/rejected": -2.449899196624756,
+      "logps/chosen": -0.5637143850326538,
+      "logps/rejected": -0.567409336566925,
+      "loss": 1.5617,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.5637143850326538,
+      "rewards/margins": 0.0036949075292795897,
+      "rewards/rejected": -0.567409336566925,
+      "step": 15
+    },
+    {
+      "epoch": 0.042866711319490956,
+      "grad_norm": 17.524849197474367,
+      "learning_rate": 2.127659574468085e-07,
+      "logits/chosen": -2.384251117706299,
+      "logits/rejected": -2.3945975303649902,
+      "logps/chosen": -0.5456732511520386,
+      "logps/rejected": -0.5427749752998352,
+      "loss": 1.5321,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.5456732511520386,
+      "rewards/margins": -0.002898297505453229,
+      "rewards/rejected": -0.5427749752998352,
+      "step": 20
+    },
+    {
+      "epoch": 0.0535833891493637,
+      "grad_norm": 14.321312880487431,
+      "learning_rate": 2.659574468085106e-07,
+      "logits/chosen": -2.279252529144287,
+      "logits/rejected": -2.2811872959136963,
+      "logps/chosen": -0.5260757803916931,
+      "logps/rejected": -0.543364405632019,
+      "loss": 1.53,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.5260757803916931,
+      "rewards/margins": 0.017288658767938614,
+      "rewards/rejected": -0.543364405632019,
+      "step": 25
+    },
+    {
+      "epoch": 0.06430006697923643,
+      "grad_norm": 19.544853384077058,
+      "learning_rate": 3.1914893617021275e-07,
+      "logits/chosen": -2.3907198905944824,
+      "logits/rejected": -2.403733015060425,
+      "logps/chosen": -0.5487326979637146,
+      "logps/rejected": -0.5386630892753601,
+      "loss": 1.5515,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.5487326979637146,
+      "rewards/margins": -0.010069593787193298,
+      "rewards/rejected": -0.5386630892753601,
+      "step": 30
+    },
+    {
+      "epoch": 0.07501674480910918,
+      "grad_norm": 24.103155081211728,
+      "learning_rate": 3.7234042553191484e-07,
+      "logits/chosen": -2.452573299407959,
+      "logits/rejected": -2.423945665359497,
+      "logps/chosen": -0.5365626811981201,
+      "logps/rejected": -0.5920503735542297,
+      "loss": 1.5478,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.5365626811981201,
+      "rewards/margins": 0.055487751960754395,
+      "rewards/rejected": -0.5920503735542297,
+      "step": 35
+    },
+    {
+      "epoch": 0.08573342263898191,
+      "grad_norm": 21.06293839287325,
+      "learning_rate": 4.25531914893617e-07,
+      "logits/chosen": -2.3676323890686035,
+      "logits/rejected": -2.349027156829834,
+      "logps/chosen": -0.5672295093536377,
+      "logps/rejected": -0.5381959676742554,
+      "loss": 1.5581,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.5672295093536377,
+      "rewards/margins": -0.02903353050351143,
+      "rewards/rejected": -0.5381959676742554,
+      "step": 40
+    },
+    {
+      "epoch": 0.09645010046885466,
+      "grad_norm": 17.138121888291746,
+      "learning_rate": 4.787234042553192e-07,
+      "logits/chosen": -2.334918260574341,
+      "logits/rejected": -2.3297195434570312,
+      "logps/chosen": -0.5420322418212891,
+      "logps/rejected": -0.560600757598877,
+      "loss": 1.5426,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5420322418212891,
+      "rewards/margins": 0.01856859028339386,
+      "rewards/rejected": -0.560600757598877,
+      "step": 45
+    },
+    {
+      "epoch": 0.1071667782987274,
+      "grad_norm": 16.650052527651823,
+      "learning_rate": 4.999367579541646e-07,
+      "logits/chosen": -2.3611702919006348,
+      "logits/rejected": -2.3566925525665283,
+      "logps/chosen": -0.542779803276062,
+      "logps/rejected": -0.5264804363250732,
+      "loss": 1.5287,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.542779803276062,
+      "rewards/margins": -0.016299355775117874,
+      "rewards/rejected": -0.5264804363250732,
+      "step": 50
+    },
+    {
+      "epoch": 0.11788345612860013,
+      "grad_norm": 25.231872930806986,
+      "learning_rate": 4.99550394651012e-07,
+      "logits/chosen": -2.4198124408721924,
+      "logits/rejected": -2.4228649139404297,
+      "logps/chosen": -0.5739614367485046,
+      "logps/rejected": -0.5911067128181458,
+      "loss": 1.5407,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.5739614367485046,
+      "rewards/margins": 0.017145240679383278,
+      "rewards/rejected": -0.5911067128181458,
+      "step": 55
+    },
+    {
+      "epoch": 0.12860013395847286,
+      "grad_norm": 27.654893603033948,
+      "learning_rate": 4.988133448023071e-07,
+      "logits/chosen": -2.3333356380462646,
+      "logits/rejected": -2.342374324798584,
+      "logps/chosen": -0.505652129650116,
+      "logps/rejected": -0.52570641040802,
+      "loss": 1.5384,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.505652129650116,
+      "rewards/margins": 0.020054226741194725,
+      "rewards/rejected": -0.52570641040802,
+      "step": 60
+    },
+    {
+      "epoch": 0.13931681178834562,
+      "grad_norm": 18.301707545724692,
+      "learning_rate": 4.977266441646379e-07,
+      "logits/chosen": -2.3869385719299316,
+      "logits/rejected": -2.391303777694702,
+      "logps/chosen": -0.5198084712028503,
+      "logps/rejected": -0.5502195358276367,
+      "loss": 1.5466,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5198084712028503,
+      "rewards/margins": 0.030411023646593094,
+      "rewards/rejected": -0.5502195358276367,
+      "step": 65
+    },
+    {
+      "epoch": 0.15003348961821836,
+      "grad_norm": 19.461976097246225,
+      "learning_rate": 4.962918198495654e-07,
+      "logits/chosen": -2.2925384044647217,
+      "logits/rejected": -2.296604633331299,
+      "logps/chosen": -0.5172106027603149,
+      "logps/rejected": -0.5665396451950073,
+      "loss": 1.5322,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5172106027603149,
+      "rewards/margins": 0.04932904243469238,
+      "rewards/rejected": -0.5665396451950073,
+      "step": 70
+    },
+    {
+      "epoch": 0.1607501674480911,
+      "grad_norm": 16.227392616134246,
+      "learning_rate": 4.945108881776135e-07,
+      "logits/chosen": -2.3317081928253174,
+      "logits/rejected": -2.3258421421051025,
+      "logps/chosen": -0.48806238174438477,
+      "logps/rejected": -0.5191291570663452,
+      "loss": 1.5323,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.48806238174438477,
+      "rewards/margins": 0.031066760420799255,
+      "rewards/rejected": -0.5191291570663452,
+      "step": 75
+    },
+    {
+      "epoch": 0.17146684527796383,
+      "grad_norm": 18.668922205376074,
+      "learning_rate": 4.923863518447879e-07,
+      "logits/chosen": -2.304633617401123,
+      "logits/rejected": -2.3267812728881836,
+      "logps/chosen": -0.4988403916358948,
+      "logps/rejected": -0.5690027475357056,
+      "loss": 1.5304,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.4988403916358948,
+      "rewards/margins": 0.07016237080097198,
+      "rewards/rejected": -0.5690027475357056,
+      "step": 80
+    },
+    {
+      "epoch": 0.18218352310783656,
+      "grad_norm": 21.78609949315303,
+      "learning_rate": 4.899211964056035e-07,
+      "logits/chosen": -2.338728427886963,
+      "logits/rejected": -2.331895112991333,
+      "logps/chosen": -0.5135633945465088,
+      "logps/rejected": -0.5660439729690552,
+      "loss": 1.5365,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5135633945465088,
+      "rewards/margins": 0.052480556070804596,
+      "rewards/rejected": -0.5660439729690552,
+      "step": 85
+    },
+    {
+      "epoch": 0.19290020093770932,
+      "grad_norm": 18.81785585703593,
+      "learning_rate": 4.871188860775643e-07,
+      "logits/chosen": -2.288607120513916,
+      "logits/rejected": -2.292832374572754,
+      "logps/chosen": -0.5013225674629211,
+      "logps/rejected": -0.5400410890579224,
+      "loss": 1.5369,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.5013225674629211,
+      "rewards/margins": 0.03871860355138779,
+      "rewards/rejected": -0.5400410890579224,
+      "step": 90
+    },
+    {
+      "epoch": 0.20361687876758205,
+      "grad_norm": 18.284986731616815,
+      "learning_rate": 4.839833588729897e-07,
+      "logits/chosen": -2.33103609085083,
+      "logits/rejected": -2.3427205085754395,
+      "logps/chosen": -0.5798801779747009,
+      "logps/rejected": -0.5994023680686951,
+      "loss": 1.5235,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.5798801779747009,
+      "rewards/margins": 0.01952212117612362,
+      "rewards/rejected": -0.5994023680686951,
+      "step": 95
+    },
+    {
+      "epoch": 0.2143335565974548,
+      "grad_norm": 24.996484906718788,
+      "learning_rate": 4.80519021065031e-07,
+      "logits/chosen": -2.3483448028564453,
+      "logits/rejected": -2.3573968410491943,
+      "logps/chosen": -0.5851219296455383,
+      "logps/rejected": -0.6033455729484558,
+      "loss": 1.5464,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.5851219296455383,
+      "rewards/margins": 0.018223673105239868,
+      "rewards/rejected": -0.6033455729484558,
+      "step": 100
+    },
+    {
+      "epoch": 0.22505023442732752,
+      "grad_norm": 15.841869984614812,
+      "learning_rate": 4.767307409956528e-07,
+      "logits/chosen": -2.3424832820892334,
+      "logits/rejected": -2.342660903930664,
+      "logps/chosen": -0.5200175046920776,
+      "logps/rejected": -0.5671975612640381,
+      "loss": 1.5177,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.5200175046920776,
+      "rewards/margins": 0.04718000069260597,
+      "rewards/rejected": -0.5671975612640381,
+      "step": 105
+    },
+    {
+      "epoch": 0.23576691225720026,
+      "grad_norm": 26.162342495475848,
+      "learning_rate": 4.726238422342805e-07,
+      "logits/chosen": -2.370981216430664,
+      "logits/rejected": -2.377699375152588,
+      "logps/chosen": -0.5119336247444153,
+      "logps/rejected": -0.5585616230964661,
+      "loss": 1.5421,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5119336247444153,
+      "rewards/margins": 0.04662800580263138,
+      "rewards/rejected": -0.5585616230964661,
+      "step": 110
+    },
+    {
+      "epoch": 0.24648359008707302,
+      "grad_norm": 22.223934331525054,
+      "learning_rate": 4.6820409609673027e-07,
+      "logits/chosen": -2.445955276489258,
+      "logits/rejected": -2.439049005508423,
+      "logps/chosen": -0.507788360118866,
+      "logps/rejected": -0.5285546779632568,
+      "loss": 1.573,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.507788360118866,
+      "rewards/margins": 0.02076634392142296,
+      "rewards/rejected": -0.5285546779632568,
+      "step": 115
+    },
+    {
+      "epoch": 0.2572002679169457,
+      "grad_norm": 17.337249756046997,
+      "learning_rate": 4.6347771353493173e-07,
+      "logits/chosen": -2.507343053817749,
+      "logits/rejected": -2.4980244636535645,
+      "logps/chosen": -0.5571922063827515,
+      "logps/rejected": -0.5488871335983276,
+      "loss": 1.5546,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.5571922063827515,
+      "rewards/margins": -0.008305097930133343,
+      "rewards/rejected": -0.5488871335983276,
+      "step": 120
+    },
+    {
+      "epoch": 0.2679169457468185,
+      "grad_norm": 18.61479199807285,
+      "learning_rate": 4.5845133640884217e-07,
+      "logits/chosen": -2.496476173400879,
+      "logits/rejected": -2.5013513565063477,
+      "logps/chosen": -0.5598434209823608,
+      "logps/rejected": -0.5717306733131409,
+      "loss": 1.5472,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.5598434209823608,
+      "rewards/margins": 0.01188725233078003,
+      "rewards/rejected": -0.5717306733131409,
+      "step": 125
+    },
+    {
+      "epoch": 0.27863362357669125,
+      "grad_norm": 24.242840152077708,
+      "learning_rate": 4.531320281528169e-07,
+      "logits/chosen": -2.577277660369873,
+      "logits/rejected": -2.5614867210388184,
+      "logps/chosen": -0.5257032513618469,
+      "logps/rejected": -0.5607699155807495,
+      "loss": 1.5206,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5257032513618469,
+      "rewards/margins": 0.03506666049361229,
+      "rewards/rejected": -0.5607699155807495,
+      "step": 130
+    },
+    {
+      "epoch": 0.289350301406564,
+      "grad_norm": 28.052518647744463,
+      "learning_rate": 4.4752726384955293e-07,
+      "logits/chosen": -2.5903048515319824,
+      "logits/rejected": -2.6012890338897705,
+      "logps/chosen": -0.5207422971725464,
+      "logps/rejected": -0.5546741485595703,
+      "loss": 1.5394,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5207422971725464,
+      "rewards/margins": 0.03393186628818512,
+      "rewards/rejected": -0.5546741485595703,
+      "step": 135
+    },
+    {
+      "epoch": 0.3000669792364367,
+      "grad_norm": 20.236382854852724,
+      "learning_rate": 4.4164491972555294e-07,
+      "logits/chosen": -2.6124188899993896,
+      "logits/rejected": -2.5999808311462402,
+      "logps/chosen": -0.5296235084533691,
+      "logps/rejected": -0.5590516924858093,
+      "loss": 1.5202,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5296235084533691,
+      "rewards/margins": 0.02942819520831108,
+      "rewards/rejected": -0.5590516924858093,
+      "step": 140
+    },
+    {
+      "epoch": 0.31078365706630945,
+      "grad_norm": 18.380103954877008,
+      "learning_rate": 4.3549326208287393e-07,
+      "logits/chosen": -2.710564613342285,
+      "logits/rejected": -2.68730092048645,
+      "logps/chosen": -0.5258200764656067,
+      "logps/rejected": -0.5745820999145508,
+      "loss": 1.5158,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.5258200764656067,
+      "rewards/margins": 0.0487620048224926,
+      "rewards/rejected": -0.5745820999145508,
+      "step": 145
+    },
+    {
+      "epoch": 0.3215003348961822,
+      "grad_norm": 19.20157418312748,
+      "learning_rate": 4.2908093568271187e-07,
+      "logits/chosen": -2.70224666595459,
+      "logits/rejected": -2.694885730743408,
+      "logps/chosen": -0.5079634189605713,
+      "logps/rejected": -0.5393082499504089,
+      "loss": 1.5455,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5079634189605713,
+      "rewards/margins": 0.03134483844041824,
+      "rewards/rejected": -0.5393082499504089,
+      "step": 150
+    },
+    {
+      "epoch": 0.3322170127260549,
+      "grad_norm": 17.41360540258233,
+      "learning_rate": 4.2241695159714843e-07,
+      "logits/chosen": -2.793240785598755,
+      "logits/rejected": -2.801339626312256,
+      "logps/chosen": -0.5376655459403992,
+      "logps/rejected": -0.5706537365913391,
+      "loss": 1.5278,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5376655459403992,
+      "rewards/margins": 0.03298817574977875,
+      "rewards/rejected": -0.5706537365913391,
+      "step": 155
+    },
+    {
+      "epoch": 0.34293369055592765,
+      "grad_norm": 21.649135642065247,
+      "learning_rate": 4.1551067454613074e-07,
+      "logits/chosen": -2.802827835083008,
+      "logits/rejected": -2.78732967376709,
+      "logps/chosen": -0.558406412601471,
+      "logps/rejected": -0.6224438548088074,
+      "loss": 1.5318,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.558406412601471,
+      "rewards/margins": 0.06403747946023941,
+      "rewards/rejected": -0.6224438548088074,
+      "step": 160
+    },
+    {
+      "epoch": 0.3536503683858004,
+      "grad_norm": 20.48435877400994,
+      "learning_rate": 4.0837180973747876e-07,
+      "logits/chosen": -2.9101483821868896,
+      "logits/rejected": -2.8818156719207764,
+      "logps/chosen": -0.5387953519821167,
+      "logps/rejected": -0.571646511554718,
+      "loss": 1.5233,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.5387953519821167,
+      "rewards/margins": 0.032851073890924454,
+      "rewards/rejected": -0.571646511554718,
+      "step": 165
+    },
+    {
+      "epoch": 0.3643670462156731,
+      "grad_norm": 17.62876339336723,
+      "learning_rate": 4.0101038922841455e-07,
+      "logits/chosen": -2.9059550762176514,
+      "logits/rejected": -2.9104220867156982,
+      "logps/chosen": -0.5236160159111023,
+      "logps/rejected": -0.5999457836151123,
+      "loss": 1.5112,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5236160159111023,
+      "rewards/margins": 0.07632973045110703,
+      "rewards/rejected": -0.5999457836151123,
+      "step": 170
+    },
+    {
+      "epoch": 0.3750837240455459,
+      "grad_norm": 20.453674701254528,
+      "learning_rate": 3.934367578277783e-07,
+      "logits/chosen": -2.9715123176574707,
+      "logits/rejected": -2.953988790512085,
+      "logps/chosen": -0.5574652552604675,
+      "logps/rejected": -0.5864735841751099,
+      "loss": 1.5348,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.5574652552604675,
+      "rewards/margins": 0.029008299112319946,
+      "rewards/rejected": -0.5864735841751099,
+      "step": 175
+    },
+    {
+      "epoch": 0.38580040187541864,
+      "grad_norm": 22.621519510630307,
+      "learning_rate": 3.856615585587434e-07,
+      "logits/chosen": -3.067772388458252,
+      "logits/rejected": -3.0693717002868652,
+      "logps/chosen": -0.5790558457374573,
+      "logps/rejected": -0.6112508177757263,
+      "loss": 1.545,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.5790558457374573,
+      "rewards/margins": 0.03219496086239815,
+      "rewards/rejected": -0.6112508177757263,
+      "step": 180
+    },
+    {
+      "epoch": 0.3965170797052914,
+      "grad_norm": 27.765398609724794,
+      "learning_rate": 3.776957177024581e-07,
+      "logits/chosen": -3.070247173309326,
+      "logits/rejected": -3.0919833183288574,
+      "logps/chosen": -0.5750337839126587,
+      "logps/rejected": -0.6032044887542725,
+      "loss": 1.5218,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.5750337839126587,
+      "rewards/margins": 0.028170805424451828,
+      "rewards/rejected": -0.6032044887542725,
+      "step": 185
+    },
+    {
+      "epoch": 0.4072337575351641,
+      "grad_norm": 19.099968753214323,
+      "learning_rate": 3.69550429443633e-07,
+      "logits/chosen": -2.944047451019287,
+      "logits/rejected": -2.959052324295044,
+      "logps/chosen": -0.5526596307754517,
+      "logps/rejected": -0.7156730890274048,
+      "loss": 1.5268,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.5526596307754517,
+      "rewards/margins": 0.16301342844963074,
+      "rewards/rejected": -0.7156730890274048,
+      "step": 190
+    },
+    {
+      "epoch": 0.41795043536503684,
+      "grad_norm": 24.556600066756868,
+      "learning_rate": 3.612371401396502e-07,
+      "logits/chosen": -3.195617198944092,
+      "logits/rejected": -3.168480396270752,
+      "logps/chosen": -0.515397846698761,
+      "logps/rejected": -0.5484700798988342,
+      "loss": 1.5305,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.515397846698761,
+      "rewards/margins": 0.03307227045297623,
+      "rewards/rejected": -0.5484700798988342,
+      "step": 195
+    },
+    {
+      "epoch": 0.4286671131949096,
+      "grad_norm": 20.445564921711252,
+      "learning_rate": 3.527675322353011e-07,
+      "logits/chosen": -3.071345567703247,
+      "logits/rejected": -3.0912113189697266,
+      "logps/chosen": -0.4999586045742035,
+      "logps/rejected": -0.5286678075790405,
+      "loss": 1.5201,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.4999586045742035,
+      "rewards/margins": 0.028709203004837036,
+      "rewards/rejected": -0.5286678075790405,
+      "step": 200
+    },
+    {
+      "epoch": 0.4393837910247823,
+      "grad_norm": 28.017877985161014,
+      "learning_rate": 3.4415350784575693e-07,
+      "logits/chosen": -3.2119345664978027,
+      "logits/rejected": -3.2098147869110107,
+      "logps/chosen": -0.48476672172546387,
+      "logps/rejected": -0.5175997018814087,
+      "loss": 1.5335,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.48476672172546387,
+      "rewards/margins": 0.032833002507686615,
+      "rewards/rejected": -0.5175997018814087,
+      "step": 205
+    },
+    {
+      "epoch": 0.45010046885465504,
+      "grad_norm": 20.446366205479023,
+      "learning_rate": 3.3540717203084224e-07,
+      "logits/chosen": -3.2167842388153076,
+      "logits/rejected": -3.209538698196411,
+      "logps/chosen": -0.5124386548995972,
+      "logps/rejected": -0.5561140179634094,
+      "loss": 1.5035,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5124386548995972,
+      "rewards/margins": 0.04367532581090927,
+      "rewards/rejected": -0.5561140179634094,
+      "step": 210
+    },
+    {
+      "epoch": 0.4608171466845278,
+      "grad_norm": 20.249310261680897,
+      "learning_rate": 3.265408157841153e-07,
+      "logits/chosen": -3.2260100841522217,
+      "logits/rejected": -3.2222793102264404,
+      "logps/chosen": -0.5612623691558838,
+      "logps/rejected": -0.6017634272575378,
+      "loss": 1.4952,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.5612623691558838,
+      "rewards/margins": 0.040501099079847336,
+      "rewards/rejected": -0.6017634272575378,
+      "step": 215
+    },
+    {
+      "epoch": 0.4715338245144005,
+      "grad_norm": 28.781450630078922,
+      "learning_rate": 3.1756689876066187e-07,
+      "logits/chosen": -3.26216459274292,
+      "logits/rejected": -3.2409801483154297,
+      "logps/chosen": -0.5308222770690918,
+      "logps/rejected": -0.609308660030365,
+      "loss": 1.5377,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.5308222770690918,
+      "rewards/margins": 0.078486368060112,
+      "rewards/rejected": -0.609308660030365,
+      "step": 220
+    },
+    {
+      "epoch": 0.4822505023442733,
+      "grad_norm": 20.9210927889319,
+      "learning_rate": 3.0849803176787184e-07,
+      "logits/chosen": -3.3171279430389404,
+      "logits/rejected": -3.3153088092803955,
+      "logps/chosen": -0.5533649921417236,
+      "logps/rejected": -0.6165200471878052,
+      "loss": 1.5082,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5533649921417236,
+      "rewards/margins": 0.06315504014492035,
+      "rewards/rejected": -0.6165200471878052,
+      "step": 225
+    },
+    {
+      "epoch": 0.49296718017414604,
+      "grad_norm": 26.953714991186636,
+      "learning_rate": 2.993469590438065e-07,
+      "logits/chosen": -3.2146403789520264,
+      "logits/rejected": -3.2017455101013184,
+      "logps/chosen": -0.539149284362793,
+      "logps/rejected": -0.5770600438117981,
+      "loss": 1.4968,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.539149284362793,
+      "rewards/margins": 0.03791072219610214,
+      "rewards/rejected": -0.5770600438117981,
+      "step": 230
+    },
+    {
+      "epoch": 0.5036838580040187,
+      "grad_norm": 23.063142701753478,
+      "learning_rate": 2.901265403480597e-07,
+      "logits/chosen": -3.2051620483398438,
+      "logits/rejected": -3.1935036182403564,
+      "logps/chosen": -0.5151172876358032,
+      "logps/rejected": -0.6125957369804382,
+      "loss": 1.508,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5151172876358032,
+      "rewards/margins": 0.09747843444347382,
+      "rewards/rejected": -0.6125957369804382,
+      "step": 235
+    },
+    {
+      "epoch": 0.5144005358338914,
+      "grad_norm": 28.679041181758652,
+      "learning_rate": 2.808497328902782e-07,
+      "logits/chosen": -3.152073621749878,
+      "logits/rejected": -3.15959095954895,
+      "logps/chosen": -0.5507190823554993,
+      "logps/rejected": -0.6257890462875366,
+      "loss": 1.5193,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5507190823554993,
+      "rewards/margins": 0.07506996393203735,
+      "rewards/rejected": -0.6257890462875366,
+      "step": 240
+    },
+    {
+      "epoch": 0.5251172136637642,
+      "grad_norm": 24.21410810508721,
+      "learning_rate": 2.715295731217396e-07,
+      "logits/chosen": -3.1899075508117676,
+      "logits/rejected": -3.1687207221984863,
+      "logps/chosen": -0.5879708528518677,
+      "logps/rejected": -0.6203640103340149,
+      "loss": 1.5033,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.5879708528518677,
+      "rewards/margins": 0.032393116503953934,
+      "rewards/rejected": -0.6203640103340149,
+      "step": 245
+    },
+    {
+      "epoch": 0.535833891493637,
+      "grad_norm": 24.944015170343583,
+      "learning_rate": 2.621791584155725e-07,
+      "logits/chosen": -3.186570167541504,
+      "logits/rejected": -3.1771557331085205,
+      "logps/chosen": -0.610142171382904,
+      "logps/rejected": -0.604741096496582,
+      "loss": 1.5341,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.610142171382904,
+      "rewards/margins": -0.005401057191193104,
+      "rewards/rejected": -0.604741096496582,
+      "step": 250
+    },
+    {
+      "epoch": 0.5465505693235098,
+      "grad_norm": 22.133595287604955,
+      "learning_rate": 2.52811628661367e-07,
+      "logits/chosen": -3.2132976055145264,
+      "logits/rejected": -3.21321439743042,
+      "logps/chosen": -0.5207762122154236,
+      "logps/rejected": -0.5928905606269836,
+      "loss": 1.4997,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5207762122154236,
+      "rewards/margins": 0.07211437076330185,
+      "rewards/rejected": -0.5928905606269836,
+      "step": 255
+    },
+    {
+      "epoch": 0.5572672471533825,
+      "grad_norm": 21.31026102730006,
+      "learning_rate": 2.434401478000353e-07,
+      "logits/chosen": -3.141242504119873,
+      "logits/rejected": -3.1591105461120605,
+      "logps/chosen": -0.526002049446106,
+      "logps/rejected": -0.6240478754043579,
+      "loss": 1.5259,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.526002049446106,
+      "rewards/margins": 0.09804578125476837,
+      "rewards/rejected": -0.6240478754043579,
+      "step": 260
+    },
+    {
+      "epoch": 0.5679839249832552,
+      "grad_norm": 25.465212058173062,
+      "learning_rate": 2.340778853248759e-07,
+      "logits/chosen": -3.1647770404815674,
+      "logits/rejected": -3.161611557006836,
+      "logps/chosen": -0.5447332859039307,
+      "logps/rejected": -0.6431838870048523,
+      "loss": 1.496,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5447332859039307,
+      "rewards/margins": 0.098450668156147,
+      "rewards/rejected": -0.6431838870048523,
+      "step": 265
+    },
+    {
+      "epoch": 0.578700602813128,
+      "grad_norm": 29.79126766513931,
+      "learning_rate": 2.247379977748339e-07,
+      "logits/chosen": -3.1814353466033936,
+      "logits/rejected": -3.1744346618652344,
+      "logps/chosen": -0.5134252905845642,
+      "logps/rejected": -0.5546069145202637,
+      "loss": 1.5172,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.5134252905845642,
+      "rewards/margins": 0.04118159040808678,
+      "rewards/rejected": -0.5546069145202637,
+      "step": 270
+    },
+    {
+      "epoch": 0.5894172806430007,
+      "grad_norm": 20.911132725985905,
+      "learning_rate": 2.1543361024596573e-07,
+      "logits/chosen": -3.2813096046447754,
+      "logits/rejected": -3.2904067039489746,
+      "logps/chosen": -0.478159099817276,
+      "logps/rejected": -0.5237951278686523,
+      "loss": 1.52,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.478159099817276,
+      "rewards/margins": 0.045636050403118134,
+      "rewards/rejected": -0.5237951278686523,
+      "step": 275
+    },
+    {
+      "epoch": 0.6001339584728734,
+      "grad_norm": 22.184303638777845,
+      "learning_rate": 2.0617779794709082e-07,
+      "logits/chosen": -3.227949857711792,
+      "logits/rejected": -3.199709415435791,
+      "logps/chosen": -0.5574294924736023,
+      "logps/rejected": -0.6131659746170044,
+      "loss": 1.5077,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5574294924736023,
+      "rewards/margins": 0.05573650076985359,
+      "rewards/rejected": -0.6131659746170044,
+      "step": 280
+    },
+    {
+      "epoch": 0.6108506363027462,
+      "grad_norm": 23.16627725406288,
+      "learning_rate": 1.9698356782554687e-07,
+      "logits/chosen": -3.2686588764190674,
+      "logits/rejected": -3.27409291267395,
+      "logps/chosen": -0.6056652069091797,
+      "logps/rejected": -0.6142518520355225,
+      "loss": 1.529,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.6056652069091797,
+      "rewards/margins": 0.008586685173213482,
+      "rewards/rejected": -0.6142518520355225,
+      "step": 285
+    },
+    {
+      "epoch": 0.6215673141326189,
+      "grad_norm": 43.63471821872903,
+      "learning_rate": 1.878638402888727e-07,
+      "logits/chosen": -3.3371078968048096,
+      "logits/rejected": -3.3204472064971924,
+      "logps/chosen": -0.5225822925567627,
+      "logps/rejected": -0.5866876840591431,
+      "loss": 1.5024,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.5225822925567627,
+      "rewards/margins": 0.06410545110702515,
+      "rewards/rejected": -0.5866876840591431,
+      "step": 290
+    },
+    {
+      "epoch": 0.6322839919624916,
+      "grad_norm": 25.701019654379248,
+      "learning_rate": 1.7883143104810223e-07,
+      "logits/chosen": -3.2457165718078613,
+      "logits/rejected": -3.2686493396759033,
+      "logps/chosen": -0.5675607919692993,
+      "logps/rejected": -0.6349104642868042,
+      "loss": 1.4993,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.5675607919692993,
+      "rewards/margins": 0.06734969466924667,
+      "rewards/rejected": -0.6349104642868042,
+      "step": 295
+    },
+    {
+      "epoch": 0.6430006697923644,
+      "grad_norm": 23.619470082480703,
+      "learning_rate": 1.6989903310818548e-07,
+      "logits/chosen": -3.3820648193359375,
+      "logits/rejected": -3.3857345581054688,
+      "logps/chosen": -0.5343354344367981,
+      "logps/rejected": -0.6022351980209351,
+      "loss": 1.4951,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.5343354344367981,
+      "rewards/margins": 0.06789974123239517,
+      "rewards/rejected": -0.6022351980209351,
+      "step": 300
+    },
+    {
+      "epoch": 0.6537173476222371,
+      "grad_norm": 22.753889982587108,
+      "learning_rate": 1.610791989308466e-07,
+      "logits/chosen": -3.3234715461730957,
+      "logits/rejected": -3.2950637340545654,
+      "logps/chosen": -0.5346772074699402,
+      "logps/rejected": -0.5783089995384216,
+      "loss": 1.5037,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5346772074699402,
+      "rewards/margins": 0.04363187402486801,
+      "rewards/rejected": -0.5783089995384216,
+      "step": 305
+    },
+    {
+      "epoch": 0.6644340254521098,
+      "grad_norm": 29.970511411150166,
+      "learning_rate": 1.523843227949418e-07,
+      "logits/chosen": -3.3544983863830566,
+      "logits/rejected": -3.355292558670044,
+      "logps/chosen": -0.5303682088851929,
+      "logps/rejected": -0.5744400024414062,
+      "loss": 1.4936,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.5303682088851929,
+      "rewards/margins": 0.04407178610563278,
+      "rewards/rejected": -0.5744400024414062,
+      "step": 310
+    },
+    {
+      "epoch": 0.6751507032819826,
+      "grad_norm": 19.95585897697304,
+      "learning_rate": 1.4382662337910867e-07,
+      "logits/chosen": -3.291701078414917,
+      "logits/rejected": -3.3025760650634766,
+      "logps/chosen": -0.5352946519851685,
+      "logps/rejected": -0.5887657403945923,
+      "loss": 1.502,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.5352946519851685,
+      "rewards/margins": 0.053471069782972336,
+      "rewards/rejected": -0.5887657403945923,
+      "step": 315
+    },
+    {
+      "epoch": 0.6858673811118553,
+      "grad_norm": 22.149800280030288,
+      "learning_rate": 1.3541812659118104e-07,
+      "logits/chosen": -3.334624767303467,
+      "logits/rejected": -3.306361675262451,
+      "logps/chosen": -0.5542501211166382,
+      "logps/rejected": -0.5587230920791626,
+      "loss": 1.524,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5542501211166382,
+      "rewards/margins": 0.004472990520298481,
+      "rewards/rejected": -0.5587230920791626,
+      "step": 320
+    },
+    {
+      "epoch": 0.696584058941728,
+      "grad_norm": 25.604347826249015,
+      "learning_rate": 1.2717064866850047e-07,
+      "logits/chosen": -3.3316922187805176,
+      "logits/rejected": -3.342952251434326,
+      "logps/chosen": -0.5008823275566101,
+      "logps/rejected": -0.5843838453292847,
+      "loss": 1.4797,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.5008823275566101,
+      "rewards/margins": 0.08350151777267456,
+      "rewards/rejected": -0.5843838453292847,
+      "step": 325
+    },
+    {
+      "epoch": 0.7073007367716008,
+      "grad_norm": 24.090724072052794,
+      "learning_rate": 1.1909577957287117e-07,
+      "logits/chosen": -3.3231868743896484,
+      "logits/rejected": -3.2961513996124268,
+      "logps/chosen": -0.5285135507583618,
+      "logps/rejected": -0.582491397857666,
+      "loss": 1.4972,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5285135507583618,
+      "rewards/margins": 0.05397792533040047,
+      "rewards/rejected": -0.582491397857666,
+      "step": 330
+    },
+    {
+      "epoch": 0.7180174146014735,
+      "grad_norm": 36.87118130194036,
+      "learning_rate": 1.1120486670349441e-07,
+      "logits/chosen": -3.4078221321105957,
+      "logits/rejected": -3.3980777263641357,
+      "logps/chosen": -0.5307896733283997,
+      "logps/rejected": -0.6150525808334351,
+      "loss": 1.5019,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.5307896733283997,
+      "rewards/margins": 0.08426298201084137,
+      "rewards/rejected": -0.6150525808334351,
+      "step": 335
+    },
+    {
+      "epoch": 0.7287340924313462,
+      "grad_norm": 25.52897276938915,
+      "learning_rate": 1.0350899895076948e-07,
+      "logits/chosen": -3.3364510536193848,
+      "logits/rejected": -3.321132183074951,
+      "logps/chosen": -0.5735017657279968,
+      "logps/rejected": -0.6429983973503113,
+      "loss": 1.521,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.5735017657279968,
+      "rewards/margins": 0.06949658691883087,
+      "rewards/rejected": -0.6429983973503113,
+      "step": 340
+    },
+    {
+      "epoch": 0.739450770261219,
+      "grad_norm": 20.331652657630677,
+      "learning_rate": 9.601899111337069e-08,
+      "logits/chosen": -3.367867946624756,
+      "logits/rejected": -3.3775951862335205,
+      "logps/chosen": -0.5771843194961548,
+      "logps/rejected": -0.6704375147819519,
+      "loss": 1.496,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.5771843194961548,
+      "rewards/margins": 0.09325330704450607,
+      "rewards/rejected": -0.6704375147819519,
+      "step": 345
+    },
+    {
+      "epoch": 0.7501674480910918,
+      "grad_norm": 22.274114959082404,
+      "learning_rate": 8.874536870049765e-08,
+      "logits/chosen": -3.3537139892578125,
+      "logits/rejected": -3.378490924835205,
+      "logps/chosen": -0.5365322828292847,
+      "logps/rejected": -0.6033391952514648,
+      "loss": 1.4971,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.5365322828292847,
+      "rewards/margins": 0.06680694222450256,
+      "rewards/rejected": -0.6033391952514648,
+      "step": 350
+    },
+    {
+      "epoch": 0.7608841259209645,
+      "grad_norm": 24.177285206790465,
+      "learning_rate": 8.169835314065663e-08,
+      "logits/chosen": -3.3511605262756348,
+      "logits/rejected": -3.3401741981506348,
+      "logps/chosen": -0.5807870626449585,
+      "logps/rejected": -0.6009158492088318,
+      "loss": 1.499,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.5807870626449585,
+      "rewards/margins": 0.020128777250647545,
+      "rewards/rejected": -0.6009158492088318,
+      "step": 355
+    },
+    {
+      "epoch": 0.7716008037508373,
+      "grad_norm": 22.05484832098868,
+      "learning_rate": 7.488784741775816e-08,
+      "logits/chosen": -3.3435721397399902,
+      "logits/rejected": -3.33024525642395,
+      "logps/chosen": -0.5081290006637573,
+      "logps/rejected": -0.5656172037124634,
+      "loss": 1.5079,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5081290006637573,
+      "rewards/margins": 0.057488154619932175,
+      "rewards/rejected": -0.5656172037124634,
+      "step": 360
+    },
+    {
+      "epoch": 0.78231748158071,
+      "grad_norm": 24.600234649150252,
+      "learning_rate": 6.832342215471715e-08,
+      "logits/chosen": -3.3981361389160156,
+      "logits/rejected": -3.39165997505188,
+      "logps/chosen": -0.5507721304893494,
+      "logps/rejected": -0.6153653860092163,
+      "loss": 1.5147,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.5507721304893494,
+      "rewards/margins": 0.06459329277276993,
+      "rewards/rejected": -0.6153653860092163,
+      "step": 365
+    },
+    {
+      "epoch": 0.7930341594105828,
+      "grad_norm": 31.85175227986763,
+      "learning_rate": 6.201430216410985e-08,
+      "logits/chosen": -3.391742706298828,
+      "logits/rejected": -3.413292646408081,
+      "logps/chosen": -0.5624402761459351,
+      "logps/rejected": -0.6359676122665405,
+      "loss": 1.5149,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5624402761459351,
+      "rewards/margins": 0.07352729141712189,
+      "rewards/rejected": -0.6359676122665405,
+      "step": 370
+    },
+    {
+      "epoch": 0.8037508372404555,
+      "grad_norm": 23.503804219453023,
+      "learning_rate": 5.596935348479029e-08,
+      "logits/chosen": -3.3563759326934814,
+      "logits/rejected": -3.338204860687256,
+      "logps/chosen": -0.6166857481002808,
+      "logps/rejected": -0.700817883014679,
+      "loss": 1.5014,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.6166857481002808,
+      "rewards/margins": 0.08413223922252655,
+      "rewards/rejected": -0.700817883014679,
+      "step": 375
+    },
+    {
+      "epoch": 0.8144675150703282,
+      "grad_norm": 26.081587652629086,
+      "learning_rate": 5.01970709226808e-08,
+      "logits/chosen": -3.443342924118042,
+      "logits/rejected": -3.4111335277557373,
+      "logps/chosen": -0.5469595193862915,
+      "logps/rejected": -0.6322815418243408,
+      "loss": 1.5083,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5469595193862915,
+      "rewards/margins": 0.08532204478979111,
+      "rewards/rejected": -0.6322815418243408,
+      "step": 380
+    },
+    {
+      "epoch": 0.825184192900201,
+      "grad_norm": 24.73716667501714,
+      "learning_rate": 4.4705566113246635e-08,
+      "logits/chosen": -3.349184513092041,
+      "logits/rejected": -3.359894275665283,
+      "logps/chosen": -0.5703479647636414,
+      "logps/rejected": -0.6489995718002319,
+      "loss": 1.4984,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.5703479647636414,
+      "rewards/margins": 0.07865166664123535,
+      "rewards/rejected": -0.6489995718002319,
+      "step": 385
+    },
+    {
+      "epoch": 0.8359008707300737,
+      "grad_norm": 24.765231706983972,
+      "learning_rate": 3.950255612243042e-08,
+      "logits/chosen": -3.3900883197784424,
+      "logits/rejected": -3.3493354320526123,
+      "logps/chosen": -0.6082748174667358,
+      "logps/rejected": -0.6323951482772827,
+      "loss": 1.5319,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.6082748174667358,
+      "rewards/margins": 0.024120373651385307,
+      "rewards/rejected": -0.6323951482772827,
+      "step": 390
+    },
+    {
+      "epoch": 0.8466175485599464,
+      "grad_norm": 23.65912516379117,
+      "learning_rate": 3.459535260206384e-08,
+      "logits/chosen": -3.3917059898376465,
+      "logits/rejected": -3.3759236335754395,
+      "logps/chosen": -0.5519801378250122,
+      "logps/rejected": -0.5884855389595032,
+      "loss": 1.5143,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.5519801378250122,
+      "rewards/margins": 0.036505263298749924,
+      "rewards/rejected": -0.5884855389595032,
+      "step": 395
+    },
+    {
+      "epoch": 0.8573342263898192,
+      "grad_norm": 28.498252560499747,
+      "learning_rate": 2.999085151499764e-08,
+      "logits/chosen": -3.3536014556884766,
+      "logits/rejected": -3.322979688644409,
+      "logps/chosen": -0.5050632953643799,
+      "logps/rejected": -0.6183533072471619,
+      "loss": 1.4908,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.5050632953643799,
+      "rewards/margins": 0.11329002678394318,
+      "rewards/rejected": -0.6183533072471619,
+      "step": 400
+    },
+    {
+      "epoch": 0.8573342263898192,
+      "eval_logits/chosen": -3.4685490131378174,
+      "eval_logits/rejected": -3.4632692337036133,
+      "eval_logps/chosen": -0.5601765513420105,
+      "eval_logps/rejected": -0.6162815093994141,
+      "eval_loss": 1.522051215171814,
+      "eval_rewards/accuracies": 0.5797872543334961,
+      "eval_rewards/chosen": -0.5601765513420105,
+      "eval_rewards/margins": 0.05610497295856476,
+      "eval_rewards/rejected": -0.6162815093994141,
+      "eval_runtime": 433.0685,
+      "eval_samples_per_second": 6.913,
+      "eval_steps_per_second": 0.434,
+      "step": 400
+    },
+    {
+      "epoch": 0.8680509042196919,
+      "grad_norm": 23.705058603217022,
+      "learning_rate": 2.5695523444387747e-08,
+      "logits/chosen": -3.364602565765381,
+      "logits/rejected": -3.350884199142456,
+      "logps/chosen": -0.5609386563301086,
+      "logps/rejected": -0.605690598487854,
+      "loss": 1.5111,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.5609386563301086,
+      "rewards/margins": 0.044751983135938644,
+      "rewards/rejected": -0.605690598487854,
+      "step": 405
+    },
+    {
+      "epoch": 0.8787675820495646,
+      "grad_norm": 22.37221809399767,
+      "learning_rate": 2.1715404500756875e-08,
+      "logits/chosen": -3.412458896636963,
+      "logits/rejected": -3.3824081420898438,
+      "logps/chosen": -0.5534378290176392,
+      "logps/rejected": -0.5746613144874573,
+      "loss": 1.4932,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.5534378290176392,
+      "rewards/margins": 0.021223513409495354,
+      "rewards/rejected": -0.5746613144874573,
+      "step": 410
+    },
+    {
+      "epoch": 0.8894842598794374,
+      "grad_norm": 26.32210656993751,
+      "learning_rate": 1.8056087839608546e-08,
+      "logits/chosen": -3.332536220550537,
+      "logits/rejected": -3.3019680976867676,
+      "logps/chosen": -0.5494991540908813,
+      "logps/rejected": -0.6182190179824829,
+      "loss": 1.4885,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5494991540908813,
+      "rewards/margins": 0.06871981918811798,
+      "rewards/rejected": -0.6182190179824829,
+      "step": 415
+    },
+    {
+      "epoch": 0.9002009377093101,
+      "grad_norm": 27.43028943315468,
+      "learning_rate": 1.4722715801513935e-08,
+      "logits/chosen": -3.3295421600341797,
+      "logits/rejected": -3.336806058883667,
+      "logps/chosen": -0.5279422998428345,
+      "logps/rejected": -0.5645946860313416,
+      "loss": 1.5217,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.5279422998428345,
+      "rewards/margins": 0.03665238991379738,
+      "rewards/rejected": -0.5645946860313416,
+      "step": 420
+    },
+    {
+      "epoch": 0.9109176155391828,
+      "grad_norm": 25.425067386156872,
+      "learning_rate": 1.1719972685717395e-08,
+      "logits/chosen": -3.337613582611084,
+      "logits/rejected": -3.3247127532958984,
+      "logps/chosen": -0.5188074111938477,
+      "logps/rejected": -0.6191619634628296,
+      "loss": 1.4926,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.5188074111938477,
+      "rewards/margins": 0.10035452991724014,
+      "rewards/rejected": -0.6191619634628296,
+      "step": 425
+    },
+    {
+      "epoch": 0.9216342933690556,
+      "grad_norm": 35.62756758017028,
+      "learning_rate": 9.05207816741485e-09,
+      "logits/chosen": -3.3888893127441406,
+      "logits/rejected": -3.4012131690979004,
+      "logps/chosen": -0.6197360754013062,
+      "logps/rejected": -0.6971450448036194,
+      "loss": 1.5011,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.6197360754013062,
+      "rewards/margins": 0.07740898430347443,
+      "rewards/rejected": -0.6971450448036194,
+      "step": 430
+    },
+    {
+      "epoch": 0.9323509711989283,
+      "grad_norm": 27.22699922034511,
+      "learning_rate": 6.722781367956481e-09,
+      "logits/chosen": -3.455385208129883,
+      "logits/rejected": -3.4597179889678955,
+      "logps/chosen": -0.5144860744476318,
+      "logps/rejected": -0.6305041909217834,
+      "loss": 1.4974,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5144860744476318,
+      "rewards/margins": 0.1160181388258934,
+      "rewards/rejected": -0.6305041909217834,
+      "step": 435
+    },
+    {
+      "epoch": 0.943067649028801,
+      "grad_norm": 23.43687138581986,
+      "learning_rate": 4.735355586305861e-09,
+      "logits/chosen": -3.2910914421081543,
+      "logits/rejected": -3.288029193878174,
+      "logps/chosen": -0.5328935980796814,
+      "logps/rejected": -0.6186143755912781,
+      "loss": 1.4731,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5328935980796814,
+      "rewards/margins": 0.08572079241275787,
+      "rewards/rejected": -0.6186143755912781,
+      "step": 440
+    },
+    {
+      "epoch": 0.9537843268586738,
+      "grad_norm": 27.25503683317219,
+      "learning_rate": 3.0925936991598454e-09,
+      "logits/chosen": -3.320718765258789,
+      "logits/rejected": -3.319967746734619,
+      "logps/chosen": -0.5472747683525085,
+      "logps/rejected": -0.6223639845848083,
+      "loss": 1.493,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.5472747683525085,
+      "rewards/margins": 0.07508920133113861,
+      "rewards/rejected": -0.6223639845848083,
+      "step": 445
+    },
+    {
+      "epoch": 0.9645010046885466,
+      "grad_norm": 23.669876406776968,
+      "learning_rate": 1.7968042361930225e-09,
+      "logits/chosen": -3.354473114013672,
+      "logits/rejected": -3.3409972190856934,
+      "logps/chosen": -0.5448306202888489,
+      "logps/rejected": -0.6130356788635254,
+      "loss": 1.4876,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.5448306202888489,
+      "rewards/margins": 0.06820502877235413,
+      "rewards/rejected": -0.6130356788635254,
+      "step": 450
+    },
+    {
+      "epoch": 0.9752176825184193,
+      "grad_norm": 27.737663302659318,
+      "learning_rate": 8.498081359423759e-10,
+      "logits/chosen": -3.331641435623169,
+      "logits/rejected": -3.3293700218200684,
+      "logps/chosen": -0.5374922156333923,
+      "logps/rejected": -0.6145657300949097,
+      "loss": 1.4953,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.5374922156333923,
+      "rewards/margins": 0.07707355916500092,
+      "rewards/rejected": -0.6145657300949097,
+      "step": 455
+    },
+    {
+      "epoch": 0.9859343603482921,
+      "grad_norm": 29.309243600698153,
+      "learning_rate": 2.5293618689056774e-10,
+      "logits/chosen": -3.3501858711242676,
+      "logits/rejected": -3.3541958332061768,
+      "logps/chosen": -0.5475316643714905,
+      "logps/rejected": -0.6366786360740662,
+      "loss": 1.5039,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5475316643714905,
+      "rewards/margins": 0.08914700895547867,
+      "rewards/rejected": -0.6366786360740662,
+      "step": 460
+    },
+    {
+      "epoch": 0.9966510381781648,
+      "grad_norm": 21.403219882354602,
+      "learning_rate": 7.02715734424153e-12,
+      "logits/chosen": -3.3826191425323486,
+      "logits/rejected": -3.36454701423645,
+      "logps/chosen": -0.5860726833343506,
+      "logps/rejected": -0.6269819736480713,
+      "loss": 1.5076,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5860726833343506,
+      "rewards/margins": 0.040909312665462494,
+      "rewards/rejected": -0.6269819736480713,
+      "step": 465
+    },
+    {
+      "epoch": 0.9987943737441393,
+      "step": 466,
+      "total_flos": 0.0,
+      "train_loss": 1.5188358733070766,
+      "train_runtime": 19352.2078,
+      "train_samples_per_second": 3.086,
+      "train_steps_per_second": 0.024
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 466,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8eb24d5e5a1d552dfd09b54b318a0bbb1b09e42a47cae9548ab65ac4c56896aa
+size 7416