Model save

README.md

@@ -0,0 +1,59 @@
+---
+base_model: YYYYYYibo/two_agent_1_epoch_2_rdpo_iter_5
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: two_agent_1_epoch_2_rdpo_iter_6
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# two_agent_1_epoch_2_rdpo_iter_6
+
+This model is a fine-tuned version of [YYYYYYibo/two_agent_1_epoch_2_rdpo_iter_5](https://huggingface.co/YYYYYYibo/two_agent_1_epoch_2_rdpo_iter_5) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-08
+- train_batch_size: 1
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 32
+- total_train_batch_size: 128
+- total_eval_batch_size: 8
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.38.2
+- Pytorch 2.3.1+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.688920441902045,
+    "train_runtime": 33857.0653,
+    "train_samples": 21135,
+    "train_samples_per_second": 0.624,
+    "train_steps_per_second": 0.005
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.38.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8ce182f7a4d0831ed3ad638f0e16331ff3c328c9b7c26e1c6e01aa64aafa2b8
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4767e1a94a2dbaacee92b54a31d0c53322125bd7a7a0883aa5dbb371e073483
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fa9c822b7b7f7be9632fff950229b899b77984ab304bfeb97f8fea8d224055d
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.688920441902045,
+    "train_runtime": 33857.0653,
+    "train_samples": 21135,
+    "train_samples_per_second": 0.624,
+    "train_steps_per_second": 0.005
+}

trainer_state.json

@@ -0,0 +1,336 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9992429977289932,
+  "eval_steps": 500,
+  "global_step": 165,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 64.33201811585948,
+      "learning_rate": 2.941176470588235e-09,
+      "logits/chosen": -1.045584797859192,
+      "logits/rejected": -1.0100939273834229,
+      "logps/chosen": -306.79974365234375,
+      "logps/pi_response": -549.74755859375,
+      "logps/ref_response": -549.74755859375,
+      "logps/rejected": -323.9404602050781,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.06,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 44.626575707422,
+      "learning_rate": 2.941176470588235e-08,
+      "logits/chosen": -0.8396223783493042,
+      "logits/rejected": -0.9049404859542847,
+      "logps/chosen": -324.0662841796875,
+      "logps/pi_response": -575.5751342773438,
+      "logps/ref_response": -575.560791015625,
+      "logps/rejected": -351.129150390625,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.00020301873155403882,
+      "rewards/margins": -0.0002261368208564818,
+      "rewards/rejected": 2.3118065655580722e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.12,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 40.54934673767081,
+      "learning_rate": 4.994932636402031e-08,
+      "logits/chosen": -0.8628154993057251,
+      "logits/rejected": -1.1046741008758545,
+      "logps/chosen": -326.1653747558594,
+      "logps/pi_response": -569.3783569335938,
+      "logps/ref_response": -569.183837890625,
+      "logps/rejected": -349.2493591308594,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.004550245590507984,
+      "rewards/margins": -0.0001727611233945936,
+      "rewards/rejected": -0.004377484787255526,
+      "step": 20
+    },
+    {
+      "epoch": 0.18,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 38.64562535999223,
+      "learning_rate": 4.905416503522123e-08,
+      "logits/chosen": -0.7020316123962402,
+      "logits/rejected": -0.8822486996650696,
+      "logps/chosen": -328.91009521484375,
+      "logps/pi_response": -573.4976806640625,
+      "logps/ref_response": -572.086669921875,
+      "logps/rejected": -355.10101318359375,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.032152824103832245,
+      "rewards/margins": 0.008358074352145195,
+      "rewards/rejected": -0.04051090031862259,
+      "step": 30
+    },
+    {
+      "epoch": 0.24,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 41.571310227946306,
+      "learning_rate": 4.707922373336523e-08,
+      "logits/chosen": -0.7425471544265747,
+      "logits/rejected": -1.000046730041504,
+      "logps/chosen": -352.6003723144531,
+      "logps/pi_response": -562.9274291992188,
+      "logps/ref_response": -560.0791625976562,
+      "logps/rejected": -367.01202392578125,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.0810643881559372,
+      "rewards/margins": 0.021482989192008972,
+      "rewards/rejected": -0.10254738479852676,
+      "step": 40
+    },
+    {
+      "epoch": 0.3,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 44.72623241432798,
+      "learning_rate": 4.4113156629677314e-08,
+      "logits/chosen": -0.83067387342453,
+      "logits/rejected": -0.9843884706497192,
+      "logps/chosen": -358.9208679199219,
+      "logps/pi_response": -579.2630615234375,
+      "logps/ref_response": -573.8562622070312,
+      "logps/rejected": -372.1846923828125,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.49687498807907104,
+      "rewards/chosen": -0.1459781974554062,
+      "rewards/margins": 0.012343029491603374,
+      "rewards/rejected": -0.15832123160362244,
+      "step": 50
+    },
+    {
+      "epoch": 0.36,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 44.14884041979375,
+      "learning_rate": 4.028910905897228e-08,
+      "logits/chosen": -0.8997384309768677,
+      "logits/rejected": -0.7802873849868774,
+      "logps/chosen": -351.44622802734375,
+      "logps/pi_response": -573.9886474609375,
+      "logps/ref_response": -567.6268310546875,
+      "logps/rejected": -372.1985168457031,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.528124988079071,
+      "rewards/chosen": -0.12361202389001846,
+      "rewards/margins": 0.016537191346287727,
+      "rewards/rejected": -0.14014920592308044,
+      "step": 60
+    },
+    {
+      "epoch": 0.42,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 49.4230332222674,
+      "learning_rate": 3.577874068920445e-08,
+      "logits/chosen": -0.88133704662323,
+      "logits/rejected": -0.7288984656333923,
+      "logps/chosen": -346.6803894042969,
+      "logps/pi_response": -582.1519165039062,
+      "logps/ref_response": -576.6680908203125,
+      "logps/rejected": -372.2113952636719,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.11654561758041382,
+      "rewards/margins": 0.022811274975538254,
+      "rewards/rejected": -0.13935688138008118,
+      "step": 70
+    },
+    {
+      "epoch": 0.48,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 47.63146957694104,
+      "learning_rate": 3.078451980100854e-08,
+      "logits/chosen": -0.843396782875061,
+      "logits/rejected": -1.014082431793213,
+      "logps/chosen": -333.9049072265625,
+      "logps/pi_response": -562.142333984375,
+      "logps/ref_response": -556.6678466796875,
+      "logps/rejected": -361.25384521484375,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.09478305280208588,
+      "rewards/margins": 0.021441372111439705,
+      "rewards/rejected": -0.11622440814971924,
+      "step": 80
+    },
+    {
+      "epoch": 0.55,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 38.6634524662906,
+      "learning_rate": 2.5530634583340587e-08,
+      "logits/chosen": -0.9618528485298157,
+      "logits/rejected": -0.7716959714889526,
+      "logps/chosen": -335.5628967285156,
+      "logps/pi_response": -553.6281127929688,
+      "logps/ref_response": -547.9144287109375,
+      "logps/rejected": -356.3701171875,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.10612684488296509,
+      "rewards/margins": 0.019539793953299522,
+      "rewards/rejected": -0.12566663324832916,
+      "step": 90
+    },
+    {
+      "epoch": 0.61,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 48.51577809039757,
+      "learning_rate": 2.0252929432814285e-08,
+      "logits/chosen": -0.8365541696548462,
+      "logits/rejected": -1.0987269878387451,
+      "logps/chosen": -341.3290710449219,
+      "logps/pi_response": -586.1930541992188,
+      "logps/ref_response": -579.7489624023438,
+      "logps/rejected": -367.02313232421875,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.09329879283905029,
+      "rewards/margins": 0.019855480641126633,
+      "rewards/rejected": -0.11315427720546722,
+      "step": 100
+    },
+    {
+      "epoch": 0.67,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 44.83795871759954,
+      "learning_rate": 1.5188318011445905e-08,
+      "logits/chosen": -0.7073591351509094,
+      "logits/rejected": -0.9396928548812866,
+      "logps/chosen": -336.29107666015625,
+      "logps/pi_response": -593.3292236328125,
+      "logps/ref_response": -587.0032958984375,
+      "logps/rejected": -365.4888916015625,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.10252567380666733,
+      "rewards/margins": 0.018411077558994293,
+      "rewards/rejected": -0.12093675136566162,
+      "step": 110
+    },
+    {
+      "epoch": 0.73,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 43.88683254658044,
+      "learning_rate": 1.0564148305586295e-08,
+      "logits/chosen": -0.8239250183105469,
+      "logits/rejected": -0.9064956903457642,
+      "logps/chosen": -337.9382019042969,
+      "logps/pi_response": -585.01220703125,
+      "logps/ref_response": -578.1124267578125,
+      "logps/rejected": -358.4151916503906,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.10588981956243515,
+      "rewards/margins": 0.012616684660315514,
+      "rewards/rejected": -0.11850650608539581,
+      "step": 120
+    },
+    {
+      "epoch": 0.79,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 41.913741627202725,
+      "learning_rate": 6.587997083462196e-09,
+      "logits/chosen": -0.7959691286087036,
+      "logits/rejected": -0.899975597858429,
+      "logps/chosen": -333.92010498046875,
+      "logps/pi_response": -568.1605224609375,
+      "logps/ref_response": -560.1768188476562,
+      "logps/rejected": -350.40264892578125,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.10813815891742706,
+      "rewards/margins": 0.024968460202217102,
+      "rewards/rejected": -0.13310661911964417,
+      "step": 130
+    },
+    {
+      "epoch": 0.85,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 54.33549512712096,
+      "learning_rate": 3.438351873250492e-09,
+      "logits/chosen": -0.8020431399345398,
+      "logits/rejected": -1.0190136432647705,
+      "logps/chosen": -331.71807861328125,
+      "logps/pi_response": -572.9759521484375,
+      "logps/ref_response": -566.2786254882812,
+      "logps/rejected": -358.4643859863281,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.10120377689599991,
+      "rewards/margins": 0.02038509212434292,
+      "rewards/rejected": -0.12158887088298798,
+      "step": 140
+    },
+    {
+      "epoch": 0.91,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 42.82795298748498,
+      "learning_rate": 1.256598743236703e-09,
+      "logits/chosen": -0.79679274559021,
+      "logits/rejected": -0.9826513528823853,
+      "logps/chosen": -328.8675231933594,
+      "logps/pi_response": -568.5698852539062,
+      "logps/ref_response": -561.6184692382812,
+      "logps/rejected": -363.5960998535156,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.10182048380374908,
+      "rewards/margins": 0.022898752242326736,
+      "rewards/rejected": -0.12471922487020493,
+      "step": 150
+    },
+    {
+      "epoch": 0.97,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 43.50550635130607,
+      "learning_rate": 1.4067554877743859e-10,
+      "logits/chosen": -0.901385486125946,
+      "logits/rejected": -0.8993357419967651,
+      "logps/chosen": -335.07171630859375,
+      "logps/pi_response": -570.0462646484375,
+      "logps/ref_response": -563.6995849609375,
+      "logps/rejected": -356.5352783203125,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.5531250238418579,
+      "rewards/chosen": -0.11258945614099503,
+      "rewards/margins": 0.02091957814991474,
+      "rewards/rejected": -0.13350901007652283,
+      "step": 160
+    },
+    {
+      "epoch": 1.0,
+      "step": 165,
+      "total_flos": 0.0,
+      "train_loss": 0.688920441902045,
+      "train_runtime": 33857.0653,
+      "train_samples_per_second": 0.624,
+      "train_steps_per_second": 0.005
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 165,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}