Model save

README.md

@@ -1,16 +1,11 @@
 ---
 license: other
-base_model: HuggingFaceH4/zephyr-7b-gemma-sft-v0.1
+base_model: Columbia-NLP/gemma-2b-zephyr-sft
 tags:
-- alignment-handbook
-- trl
-- dpo
-- generated_from_trainer
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
-datasets:
-- argilla/dpo-mix-7k
 model-index:
 - name: zephyr-7b-gemma-dpo
   results: []
@@ -21,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-gemma-dpo
 
-This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on the argilla/dpo-mix-7k dataset.
+This model is a fine-tuned version of [Columbia-NLP/gemma-2b-zephyr-sft](https://huggingface.co/Columbia-NLP/gemma-2b-zephyr-sft) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4683
-- Rewards/chosen: -3.0221
-- Rewards/rejected: -4.6813
-- Rewards/accuracies: 0.7708
-- Rewards/margins: 1.6592
-- Logps/rejected: -453.7782
-- Logps/chosen: -423.6228
-- Logits/rejected: 90.9822
-- Logits/chosen: 96.7158
+- Loss: 0.5972
+- Rewards/chosen: 0.3534
+- Rewards/rejected: 0.1096
+- Rewards/accuracies: 0.6771
+- Rewards/margins: 0.2437
+- Logps/rejected: -373.4094
+- Logps/chosen: -372.0067
+- Logits/rejected: -12.6654
+- Logits/chosen: -13.2926
 
 ## Model description
 
@@ -68,7 +63,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.1591        | 1.8957 | 100  | 0.4692          | -2.9810        | -4.6451          | 0.7604             | 1.6641          | -453.0530      | -422.7995    | 91.0574         | 96.7661       |
+| 0.5732        | 1.8957 | 100  | 0.5972          | 0.3534         | 0.1096           | 0.6771             | 0.2437          | -373.4094      | -372.0067    | -12.6654        | -13.2926      |
 
 
 ### Framework versions

all_results.json

@@ -1,22 +1,22 @@
 {
     "epoch": 1.971563981042654,
-    "eval_logits/chosen": 96.71578216552734,
-    "eval_logits/rejected": 90.98221588134766,
-    "eval_logps/chosen": -423.6227722167969,
-    "eval_logps/rejected": -453.7782287597656,
-    "eval_loss": 0.468290776014328,
-    "eval_rewards/accuracies": 0.7708333134651184,
-    "eval_rewards/chosen": -3.0221338272094727,
-    "eval_rewards/margins": 1.6591955423355103,
-    "eval_rewards/rejected": -4.681329727172852,
-    "eval_runtime": 58.6185,
+    "eval_logits/chosen": 96.0078125,
+    "eval_logits/rejected": 90.1099853515625,
+    "eval_logps/chosen": -434.6161193847656,
+    "eval_logps/rejected": -466.3072814941406,
+    "eval_loss": 0.4578173756599426,
+    "eval_rewards/accuracies": 0.7604166865348816,
+    "eval_rewards/chosen": -3.5473945140838623,
+    "eval_rewards/margins": 1.6644223928451538,
+    "eval_rewards/rejected": -5.211816787719727,
+    "eval_runtime": 124.9704,
     "eval_samples": 750,
-    "eval_samples_per_second": 12.795,
-    "eval_steps_per_second": 0.409,
+    "eval_samples_per_second": 6.001,
+    "eval_steps_per_second": 0.192,
     "total_flos": 0.0,
-    "train_loss": 0.3921648321243433,
-    "train_runtime": 1190.3032,
+    "train_loss": 0.627926590350958,
+    "train_runtime": 756.4701,
     "train_samples": 6750,
-    "train_samples_per_second": 11.342,
-    "train_steps_per_second": 0.087
+    "train_samples_per_second": 17.846,
+    "train_steps_per_second": 0.137
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "HuggingFaceH4/zephyr-7b-gemma-sft-v0.1",
+  "_name_or_path": "Columbia-NLP/gemma-2b-zephyr-sft",
   "architectures": [
     "GemmaForCausalLM"
   ],
@@ -10,20 +10,20 @@
   "head_dim": 256,
   "hidden_act": "gelu",
   "hidden_activation": null,
-  "hidden_size": 3072,
+  "hidden_size": 2048,
   "initializer_range": 0.02,
-  "intermediate_size": 24576,
+  "intermediate_size": 16384,
   "max_position_embeddings": 8192,
   "model_type": "gemma",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 28,
-  "num_key_value_heads": 16,
+  "num_attention_heads": 8,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 1,
   "pad_token_id": 0,
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.1",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 256000
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.971563981042654,
-    "eval_logits/chosen": 96.71578216552734,
-    "eval_logits/rejected": 90.98221588134766,
-    "eval_logps/chosen": -423.6227722167969,
-    "eval_logps/rejected": -453.7782287597656,
-    "eval_loss": 0.468290776014328,
-    "eval_rewards/accuracies": 0.7708333134651184,
-    "eval_rewards/chosen": -3.0221338272094727,
-    "eval_rewards/margins": 1.6591955423355103,
-    "eval_rewards/rejected": -4.681329727172852,
-    "eval_runtime": 58.6185,
+    "eval_logits/chosen": 96.0078125,
+    "eval_logits/rejected": 90.1099853515625,
+    "eval_logps/chosen": -434.6161193847656,
+    "eval_logps/rejected": -466.3072814941406,
+    "eval_loss": 0.4578173756599426,
+    "eval_rewards/accuracies": 0.7604166865348816,
+    "eval_rewards/chosen": -3.5473945140838623,
+    "eval_rewards/margins": 1.6644223928451538,
+    "eval_rewards/rejected": -5.211816787719727,
+    "eval_runtime": 124.9704,
     "eval_samples": 750,
-    "eval_samples_per_second": 12.795,
-    "eval_steps_per_second": 0.409
+    "eval_samples_per_second": 6.001,
+    "eval_steps_per_second": 0.192
 }

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe8c7b38eceaa5e7aeae3e932e2354f0dea3d47e873adf9bc1c35c4e2da99f2
+size 4945242264

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d618cb86564d9ea380614a6c12b636c502fb5ff187be1cb9426352eb6e895f51
+size 67121608

model.safetensors.index.json

@@ -1,261 +1,171 @@
 {
   "metadata": {
-    "total_size": 17075361792
+    "total_size": 5012344832
   },
   "weight_map": {
-    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.norm.weight": "model-00004-of-00004.safetensors"
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
   }
 }

runs/Apr27_14-56-02_660111d13776/events.out.tfevents.1714226380.660111d13776.25904.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:405d736d5884f25ea2c9a0384d1867c1f511cb06a4e1d8b83f1c865f03e3e449
+size 13438

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.971563981042654,
     "total_flos": 0.0,
-    "train_loss": 0.3921648321243433,
-    "train_runtime": 1190.3032,
+    "train_loss": 0.627926590350958,
+    "train_runtime": 756.4701,
     "train_samples": 6750,
-    "train_samples_per_second": 11.342,
-    "train_steps_per_second": 0.087
+    "train_samples_per_second": 17.846,
+    "train_steps_per_second": 0.137
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.018957345971563982,
-      "grad_norm": 133.64647421295854,
+      "grad_norm": 15.786988646394411,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": 117.4909439086914,
-      "logits/rejected": 126.8502426147461,
-      "logps/chosen": -338.3250732421875,
-      "logps/rejected": -438.210205078125,
+      "logits/chosen": -13.905267715454102,
+      "logits/rejected": -14.118387222290039,
+      "logps/chosen": -350.8895263671875,
+      "logps/rejected": -446.6286926269531,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,178 +25,178 @@
     },
     {
       "epoch": 0.1895734597156398,
-      "grad_norm": 137.17714765050428,
+      "grad_norm": 15.908099576913655,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": 135.0139923095703,
-      "logits/rejected": 138.361328125,
-      "logps/chosen": -397.126220703125,
-      "logps/rejected": -439.42083740234375,
-      "loss": 0.7143,
-      "rewards/accuracies": 0.3888888955116272,
-      "rewards/chosen": -0.02168009988963604,
-      "rewards/margins": -0.04445798695087433,
-      "rewards/rejected": 0.02277788519859314,
+      "logits/chosen": -14.040081024169922,
+      "logits/rejected": -14.157392501831055,
+      "logps/chosen": -416.2701416015625,
+      "logps/rejected": -449.4697265625,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5138888955116272,
+      "rewards/chosen": 0.0006088384543545544,
+      "rewards/margins": 0.008209776133298874,
+      "rewards/rejected": -0.007600938435643911,
       "step": 10
     },
     {
       "epoch": 0.3791469194312796,
-      "grad_norm": 125.18497680199994,
+      "grad_norm": 14.229474825008781,
       "learning_rate": 4.885348141000122e-07,
-      "logits/chosen": 121.53276062011719,
-      "logits/rejected": 125.26307678222656,
-      "logps/chosen": -370.91107177734375,
-      "logps/rejected": -423.38677978515625,
-      "loss": 0.6418,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.08632902055978775,
-      "rewards/margins": 0.28587669134140015,
-      "rewards/rejected": -0.199547678232193,
+      "logits/chosen": -13.39338207244873,
+      "logits/rejected": -13.542058944702148,
+      "logps/chosen": -392.9753723144531,
+      "logps/rejected": -427.68096923828125,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.010071685537695885,
+      "rewards/margins": 0.003802267834544182,
+      "rewards/rejected": 0.006269416771829128,
       "step": 20
     },
     {
       "epoch": 0.5687203791469194,
-      "grad_norm": 115.98151525592598,
+      "grad_norm": 15.853985724357454,
       "learning_rate": 4.5025027361734613e-07,
-      "logits/chosen": 142.67178344726562,
-      "logits/rejected": 136.16537475585938,
-      "logps/chosen": -415.01104736328125,
-      "logps/rejected": -460.3519592285156,
-      "loss": 0.5673,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -1.1033741235733032,
-      "rewards/margins": 0.8893669843673706,
-      "rewards/rejected": -1.9927412271499634,
+      "logits/chosen": -14.269427299499512,
+      "logits/rejected": -13.808093070983887,
+      "logps/chosen": -412.9443359375,
+      "logps/rejected": -428.38494873046875,
+      "loss": 0.674,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.04771440848708153,
+      "rewards/margins": 0.035354893654584885,
+      "rewards/rejected": 0.012359511107206345,
       "step": 30
     },
     {
       "epoch": 0.7582938388625592,
-      "grad_norm": 112.04677171325864,
+      "grad_norm": 14.687978809678542,
       "learning_rate": 3.893311157806091e-07,
-      "logits/chosen": 124.56459045410156,
-      "logits/rejected": 113.08979797363281,
-      "logps/chosen": -391.10174560546875,
-      "logps/rejected": -417.09051513671875,
-      "loss": 0.5586,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.8466203212738037,
-      "rewards/margins": 1.0624934434890747,
-      "rewards/rejected": -2.909113883972168,
+      "logits/chosen": -13.886492729187012,
+      "logits/rejected": -13.28197956085205,
+      "logps/chosen": -374.98211669921875,
+      "logps/rejected": -366.5968322753906,
+      "loss": 0.657,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.13442906737327576,
+      "rewards/margins": 0.07902240008115768,
+      "rewards/rejected": 0.05540664866566658,
       "step": 40
     },
     {
       "epoch": 0.9478672985781991,
-      "grad_norm": 126.0267199667638,
+      "grad_norm": 15.872142673244408,
       "learning_rate": 3.126631330646801e-07,
-      "logits/chosen": 138.824462890625,
-      "logits/rejected": 142.9259033203125,
-      "logps/chosen": -455.6646423339844,
-      "logps/rejected": -536.987548828125,
-      "loss": 0.4941,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.8371152877807617,
-      "rewards/margins": 1.2058273553848267,
-      "rewards/rejected": -3.042942523956299,
+      "logits/chosen": -14.917936325073242,
+      "logits/rejected": -14.90648078918457,
+      "logps/chosen": -429.6836853027344,
+      "logps/rejected": -480.3504943847656,
+      "loss": 0.6344,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.24091288447380066,
+      "rewards/margins": 0.1330389827489853,
+      "rewards/rejected": 0.10787389427423477,
       "step": 50
     },
     {
       "epoch": 1.1374407582938388,
-      "grad_norm": 63.02753605606795,
+      "grad_norm": 14.061428605486398,
       "learning_rate": 2.2891223348923882e-07,
-      "logits/chosen": 131.37802124023438,
-      "logits/rejected": 134.72222900390625,
-      "logps/chosen": -436.17047119140625,
-      "logps/rejected": -520.2355346679688,
-      "loss": 0.3078,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": -1.9126123189926147,
-      "rewards/margins": 2.285891056060791,
-      "rewards/rejected": -4.198503017425537,
+      "logits/chosen": -14.622962951660156,
+      "logits/rejected": -14.403157234191895,
+      "logps/chosen": -415.7464904785156,
+      "logps/rejected": -441.731201171875,
+      "loss": 0.6063,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.3395090103149414,
+      "rewards/margins": 0.22218124568462372,
+      "rewards/rejected": 0.11732780933380127,
       "step": 60
     },
     {
       "epoch": 1.3270142180094786,
-      "grad_norm": 59.15589622996558,
+      "grad_norm": 12.963152293888875,
       "learning_rate": 1.4754491880085317e-07,
-      "logits/chosen": 124.51689147949219,
-      "logits/rejected": 126.70524597167969,
-      "logps/chosen": -402.62066650390625,
-      "logps/rejected": -505.50006103515625,
-      "loss": 0.1932,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": -1.7549495697021484,
-      "rewards/margins": 2.8932533264160156,
-      "rewards/rejected": -4.648203372955322,
+      "logits/chosen": -14.022384643554688,
+      "logits/rejected": -13.828951835632324,
+      "logps/chosen": -382.23468017578125,
+      "logps/rejected": -418.2818908691406,
+      "loss": 0.6011,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.3396778702735901,
+      "rewards/margins": 0.22157195210456848,
+      "rewards/rejected": 0.118105947971344,
       "step": 70
     },
     {
       "epoch": 1.5165876777251186,
-      "grad_norm": 44.24206971141979,
+      "grad_norm": 12.394681314131397,
       "learning_rate": 7.775827023107834e-08,
-      "logits/chosen": 111.74947357177734,
-      "logits/rejected": 128.5332489013672,
-      "logps/chosen": -400.06146240234375,
-      "logps/rejected": -519.4473876953125,
-      "loss": 0.1687,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": -2.326984167098999,
-      "rewards/margins": 2.937407970428467,
-      "rewards/rejected": -5.264392375946045,
+      "logits/chosen": -13.705121040344238,
+      "logits/rejected": -14.205709457397461,
+      "logps/chosen": -367.263427734375,
+      "logps/rejected": -423.30841064453125,
+      "loss": 0.5788,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.36119210720062256,
+      "rewards/margins": 0.3365553319454193,
+      "rewards/rejected": 0.024636749178171158,
       "step": 80
     },
     {
       "epoch": 1.7061611374407581,
-      "grad_norm": 50.07584592888485,
+      "grad_norm": 14.456589635016153,
       "learning_rate": 2.7440387297912122e-08,
-      "logits/chosen": 110.84814453125,
-      "logits/rejected": 123.78230285644531,
-      "logps/chosen": -435.03265380859375,
-      "logps/rejected": -550.7723388671875,
-      "loss": 0.1579,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -2.400338649749756,
-      "rewards/margins": 3.238767623901367,
-      "rewards/rejected": -5.639105796813965,
+      "logits/chosen": -13.98394775390625,
+      "logits/rejected": -14.161648750305176,
+      "logps/chosen": -399.45458984375,
+      "logps/rejected": -447.48828125,
+      "loss": 0.5766,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.3995341658592224,
+      "rewards/margins": 0.34082064032554626,
+      "rewards/rejected": 0.05871356278657913,
       "step": 90
     },
     {
       "epoch": 1.8957345971563981,
-      "grad_norm": 49.65552371508206,
+      "grad_norm": 13.44211674398592,
       "learning_rate": 2.27878296044029e-09,
-      "logits/chosen": 117.1094970703125,
-      "logits/rejected": 117.060302734375,
-      "logps/chosen": -427.23431396484375,
-      "logps/rejected": -520.2066650390625,
-      "loss": 0.1591,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -2.2608113288879395,
-      "rewards/margins": 2.9113571643829346,
-      "rewards/rejected": -5.172169208526611,
+      "logits/chosen": -14.160197257995605,
+      "logits/rejected": -14.141824722290039,
+      "logps/chosen": -392.3072509765625,
+      "logps/rejected": -421.604248046875,
+      "loss": 0.5732,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.4029002785682678,
+      "rewards/margins": 0.27652695775032043,
+      "rewards/rejected": 0.1263733208179474,
       "step": 100
     },
     {
       "epoch": 1.8957345971563981,
-      "eval_logits/chosen": 96.76607513427734,
-      "eval_logits/rejected": 91.05736541748047,
-      "eval_logps/chosen": -422.7994689941406,
-      "eval_logps/rejected": -453.052978515625,
-      "eval_loss": 0.4691648781299591,
-      "eval_rewards/accuracies": 0.7604166865348816,
-      "eval_rewards/chosen": -2.9809672832489014,
-      "eval_rewards/margins": 1.6640973091125488,
-      "eval_rewards/rejected": -4.645064353942871,
-      "eval_runtime": 56.9732,
-      "eval_samples_per_second": 13.164,
-      "eval_steps_per_second": 0.421,
+      "eval_logits/chosen": -13.292621612548828,
+      "eval_logits/rejected": -12.66539478302002,
+      "eval_logps/chosen": -372.0066833496094,
+      "eval_logps/rejected": -373.4093933105469,
+      "eval_loss": 0.5971602201461792,
+      "eval_rewards/accuracies": 0.6770833134651184,
+      "eval_rewards/chosen": 0.3533553183078766,
+      "eval_rewards/margins": 0.24372106790542603,
+      "eval_rewards/rejected": 0.10963428020477295,
+      "eval_runtime": 20.0916,
+      "eval_samples_per_second": 37.329,
+      "eval_steps_per_second": 1.195,
       "step": 100
     },
     {
       "epoch": 1.971563981042654,
       "step": 104,
       "total_flos": 0.0,
-      "train_loss": 0.3921648321243433,
-      "train_runtime": 1190.3032,
-      "train_samples_per_second": 11.342,
-      "train_steps_per_second": 0.087
+      "train_loss": 0.627926590350958,
+      "train_runtime": 756.4701,
+      "train_samples_per_second": 17.846,
+      "train_steps_per_second": 0.137
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:912545168b662aa28a59d2ea7ff1a88af69cd74e0b822da195076b8d4f0f07e5
+oid sha256:c83a4cd67c56c86e8779774ef2b3c0c2d20d775dd7b0aa4eba03778d916c3903
 size 6264