End of training

README.md

@@ -0,0 +1,78 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: mistral-orpo-mix-beta
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral-orpo-mix-beta
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.2152
+- Rewards/chosen: -0.1113
+- Rewards/rejected: -0.1461
+- Rewards/accuracies: 0.6562
+- Rewards/margins: 0.0348
+- Logps/rejected: -1.4607
+- Logps/chosen: -1.1130
+- Logits/rejected: -2.4201
+- Logits/chosen: -2.4178
+- Nll Loss: 1.1711
+- Log Odds Ratio: -0.6108
+- Log Odds Chosen: 0.5395
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: inverse_sqrt
+- lr_scheduler_warmup_ratio: 0.1
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Nll Loss | Log Odds Ratio | Log Odds Chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|:--------------:|:---------------:|
+| 0.9961        | 1.0   | 211  | 0.9199          | -0.0831        | -0.1055          | 0.6302             | 0.0224          | -1.0547        | -0.8311      | -2.3542         | -2.3590       | 0.8774   | -0.6294        | 0.3896          |
+| 0.4999        | 2.0   | 422  | 0.9813          | -0.0894        | -0.1184          | 0.6615             | 0.0290          | -1.1836        | -0.8937      | -2.5184         | -2.5177       | 0.9404   | -0.6012        | 0.4994          |
+| 0.1899        | 3.0   | 633  | 1.2152          | -0.1113        | -0.1461          | 0.6562             | 0.0348          | -1.4607        | -1.1130      | -2.4201         | -2.4178       | 1.1711   | -0.6108        | 0.5395          |
+
+
+### Framework versions
+
+- Transformers 4.39.0
+- Pytorch 2.1.1+cu121
+- Datasets 2.16.1
+- Tokenizers 0.15.2

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|im_end|>": 32001,
+  "<|im_start|>": 32000
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.0",
+  "use_cache": true,
+  "vocab_size": 32002
+}

final_checkpoint/added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|im_end|>": 32001,
+  "<|im_start|>": 32000
+}

final_checkpoint/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.0",
+  "use_cache": true,
+  "vocab_size": 32002
+}

final_checkpoint/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

final_checkpoint/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

final_checkpoint/tokenizer_config.json

@@ -0,0 +1,62 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 32000,
+  "eos_token_id": 32001,
+  "pad_token_id": 32001,
+  "transformers_version": "4.39.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba9dfcf688bfdd55a8f711a646014d65ed31c1855ea4b247eea69677f63ab26e
+size 4943178720

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0d8c0fc0a340374dfddd590f5070f60dfa480394a08b1c6e94378a339f1c3d9
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:734f7e2a0baefd53bac6d6d88304db9295b9a9f892879ebcd3900d3990d059ea
+size 4540532728

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483496960
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Mar26_03-51-15_alvarobartt-kind-frog-1-0/events.out.tfevents.1711439538.alvarobartt-kind-frog-1-0.445.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9026aed9b1566f975b01ce90693209832a3d4d347d152972bd92f768994bb7ef
+size 62089

special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,62 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

trainer_state.json

@@ -0,0 +1,1221 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 633,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.05,
+      "grad_norm": 45.76888063592087,
+      "learning_rate": 5.000000000000001e-07,
+      "log_odds_chosen": 0.2575085163116455,
+      "log_odds_ratio": -0.704267144203186,
+      "logits/chosen": -2.8495709896087646,
+      "logits/rejected": -2.8519797325134277,
+      "logps/chosen": -1.2749807834625244,
+      "logps/rejected": -1.5134317874908447,
+      "loss": 1.2045,
+      "nll_loss": 1.1489894390106201,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.127498060464859,
+      "rewards/margins": 0.023845110088586807,
+      "rewards/rejected": -0.1513431817293167,
+      "step": 10
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 17.550572571834728,
+      "learning_rate": 1.0000000000000002e-06,
+      "log_odds_chosen": 0.18295660614967346,
+      "log_odds_ratio": -0.7083484530448914,
+      "logits/chosen": -2.9011380672454834,
+      "logits/rejected": -2.903749465942383,
+      "logps/chosen": -0.9543957710266113,
+      "logps/rejected": -1.0926854610443115,
+      "loss": 1.0092,
+      "nll_loss": 0.9102001190185547,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.09543957561254501,
+      "rewards/margins": 0.013828963041305542,
+      "rewards/rejected": -0.10926854610443115,
+      "step": 20
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 10.772595379798407,
+      "learning_rate": 1.5e-06,
+      "log_odds_chosen": 0.22791412472724915,
+      "log_odds_ratio": -0.6785577535629272,
+      "logits/chosen": -2.970428943634033,
+      "logits/rejected": -2.9696388244628906,
+      "logps/chosen": -0.9358822107315063,
+      "logps/rejected": -1.0665825605392456,
+      "loss": 0.968,
+      "nll_loss": 0.8899785280227661,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0935882106423378,
+      "rewards/margins": 0.013070052489638329,
+      "rewards/rejected": -0.10665826499462128,
+      "step": 30
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 11.974913883612869,
+      "learning_rate": 2.0000000000000003e-06,
+      "log_odds_chosen": 0.12087659537792206,
+      "log_odds_ratio": -0.7188054323196411,
+      "logits/chosen": -2.883781671524048,
+      "logits/rejected": -2.8842344284057617,
+      "logps/chosen": -0.9183737635612488,
+      "logps/rejected": -0.9950372576713562,
+      "loss": 0.9799,
+      "nll_loss": 0.8861400485038757,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.09183738380670547,
+      "rewards/margins": 0.007666352204978466,
+      "rewards/rejected": -0.09950373321771622,
+      "step": 40
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 13.063526867338696,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 0.4301052689552307,
+      "log_odds_ratio": -0.6214904189109802,
+      "logits/chosen": -2.832704782485962,
+      "logits/rejected": -2.8430252075195312,
+      "logps/chosen": -0.9002718925476074,
+      "logps/rejected": -1.2055391073226929,
+      "loss": 0.9584,
+      "nll_loss": 0.9324554204940796,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.09002719074487686,
+      "rewards/margins": 0.03052671253681183,
+      "rewards/rejected": -0.12055391073226929,
+      "step": 50
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 10.235387913194325,
+      "learning_rate": 3e-06,
+      "log_odds_chosen": 0.3391026258468628,
+      "log_odds_ratio": -0.6046633720397949,
+      "logits/chosen": -2.597712993621826,
+      "logits/rejected": -2.598874568939209,
+      "logps/chosen": -0.7375704050064087,
+      "logps/rejected": -0.9383118748664856,
+      "loss": 0.8856,
+      "nll_loss": 0.7253891229629517,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.07375704497098923,
+      "rewards/margins": 0.020074138417840004,
+      "rewards/rejected": -0.09383118897676468,
+      "step": 60
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 10.615024222922383,
+      "learning_rate": 3.5e-06,
+      "log_odds_chosen": 0.34223267436027527,
+      "log_odds_ratio": -0.6422346830368042,
+      "logits/chosen": -2.540403366088867,
+      "logits/rejected": -2.5442099571228027,
+      "logps/chosen": -0.8444018363952637,
+      "logps/rejected": -1.083935022354126,
+      "loss": 0.9384,
+      "nll_loss": 0.8686600923538208,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08444017916917801,
+      "rewards/margins": 0.02395332232117653,
+      "rewards/rejected": -0.10839351266622543,
+      "step": 70
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 10.120353883295946,
+      "learning_rate": 4.000000000000001e-06,
+      "log_odds_chosen": 0.026019399985671043,
+      "log_odds_ratio": -0.7628353834152222,
+      "logits/chosen": -2.6254281997680664,
+      "logits/rejected": -2.6229329109191895,
+      "logps/chosen": -0.8879353404045105,
+      "logps/rejected": -0.9107568860054016,
+      "loss": 0.921,
+      "nll_loss": 0.9201810956001282,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.08879353106021881,
+      "rewards/margins": 0.0022821640595793724,
+      "rewards/rejected": -0.09107569605112076,
+      "step": 80
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 9.558778662613774,
+      "learning_rate": 4.5e-06,
+      "log_odds_chosen": 0.29862236976623535,
+      "log_odds_ratio": -0.6326144933700562,
+      "logits/chosen": -2.6428685188293457,
+      "logits/rejected": -2.621931552886963,
+      "logps/chosen": -0.8801389932632446,
+      "logps/rejected": -1.0846771001815796,
+      "loss": 0.9325,
+      "nll_loss": 0.8730384707450867,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0880139023065567,
+      "rewards/margins": 0.020453810691833496,
+      "rewards/rejected": -0.1084677204489708,
+      "step": 90
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 11.437968315222852,
+      "learning_rate": 5e-06,
+      "log_odds_chosen": 0.3877044916152954,
+      "log_odds_ratio": -0.6339091658592224,
+      "logits/chosen": -2.583739757537842,
+      "logits/rejected": -2.6161398887634277,
+      "logps/chosen": -0.8836472630500793,
+      "logps/rejected": -1.1088001728057861,
+      "loss": 0.9742,
+      "nll_loss": 0.9505327939987183,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08836471289396286,
+      "rewards/margins": 0.02251528576016426,
+      "rewards/rejected": -0.11088001728057861,
+      "step": 100
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 9.765547500332794,
+      "learning_rate": 4.767312946227961e-06,
+      "log_odds_chosen": 0.33769896626472473,
+      "log_odds_ratio": -0.6257166266441345,
+      "logits/chosen": -2.6281538009643555,
+      "logits/rejected": -2.6198818683624268,
+      "logps/chosen": -0.7726272344589233,
+      "logps/rejected": -0.9787896871566772,
+      "loss": 0.9403,
+      "nll_loss": 0.8244824409484863,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07726272195577621,
+      "rewards/margins": 0.020616240799427032,
+      "rewards/rejected": -0.09787896275520325,
+      "step": 110
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 10.192809233857067,
+      "learning_rate": 4.564354645876385e-06,
+      "log_odds_chosen": 0.31468966603279114,
+      "log_odds_ratio": -0.6638727784156799,
+      "logits/chosen": -2.6107370853424072,
+      "logits/rejected": -2.601783275604248,
+      "logps/chosen": -0.8392475247383118,
+      "logps/rejected": -1.036677598953247,
+      "loss": 0.9841,
+      "nll_loss": 0.8654748797416687,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08392475545406342,
+      "rewards/margins": 0.019743015989661217,
+      "rewards/rejected": -0.10366777330636978,
+      "step": 120
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 11.038589594109835,
+      "learning_rate": 4.385290096535147e-06,
+      "log_odds_chosen": 0.2760984003543854,
+      "log_odds_ratio": -0.6513851881027222,
+      "logits/chosen": -2.566074848175049,
+      "logits/rejected": -2.565674304962158,
+      "logps/chosen": -0.833086371421814,
+      "logps/rejected": -0.9857648611068726,
+      "loss": 0.9998,
+      "nll_loss": 0.9077309370040894,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0833086371421814,
+      "rewards/margins": 0.015267851762473583,
+      "rewards/rejected": -0.09857650101184845,
+      "step": 130
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 9.191362766920426,
+      "learning_rate": 4.2257712736425835e-06,
+      "log_odds_chosen": 0.37576600909233093,
+      "log_odds_ratio": -0.6204187273979187,
+      "logits/chosen": -2.577667474746704,
+      "logits/rejected": -2.584357976913452,
+      "logps/chosen": -0.8501531481742859,
+      "logps/rejected": -1.1260815858840942,
+      "loss": 0.9545,
+      "nll_loss": 0.8476670980453491,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.08501531183719635,
+      "rewards/margins": 0.027592848986387253,
+      "rewards/rejected": -0.1126081570982933,
+      "step": 140
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 10.020928126499054,
+      "learning_rate": 4.082482904638631e-06,
+      "log_odds_chosen": 0.40555381774902344,
+      "log_odds_ratio": -0.6359796524047852,
+      "logits/chosen": -2.624826669692993,
+      "logits/rejected": -2.6218724250793457,
+      "logps/chosen": -0.8737274408340454,
+      "logps/rejected": -1.1572520732879639,
+      "loss": 0.9391,
+      "nll_loss": 0.8726488947868347,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08737273514270782,
+      "rewards/margins": 0.028352469205856323,
+      "rewards/rejected": -0.11572520434856415,
+      "step": 150
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 11.288206284049263,
+      "learning_rate": 3.952847075210474e-06,
+      "log_odds_chosen": 0.35032859444618225,
+      "log_odds_ratio": -0.6347723007202148,
+      "logits/chosen": -2.7082316875457764,
+      "logits/rejected": -2.711594820022583,
+      "logps/chosen": -0.8456757664680481,
+      "logps/rejected": -1.0942612886428833,
+      "loss": 0.9052,
+      "nll_loss": 0.8740180730819702,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0845675840973854,
+      "rewards/margins": 0.024858549237251282,
+      "rewards/rejected": -0.10942612588405609,
+      "step": 160
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 9.222481367340817,
+      "learning_rate": 3.834824944236852e-06,
+      "log_odds_chosen": 0.14469774067401886,
+      "log_odds_ratio": -0.6979460120201111,
+      "logits/chosen": -2.575446844100952,
+      "logits/rejected": -2.572465419769287,
+      "logps/chosen": -0.8792887926101685,
+      "logps/rejected": -0.9753986597061157,
+      "loss": 0.9229,
+      "nll_loss": 0.8886219263076782,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.0879288762807846,
+      "rewards/margins": 0.009610971435904503,
+      "rewards/rejected": -0.09753986448049545,
+      "step": 170
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 10.622839554119508,
+      "learning_rate": 3.72677996249965e-06,
+      "log_odds_chosen": 0.18417394161224365,
+      "log_odds_ratio": -0.7312262058258057,
+      "logits/chosen": -2.558830976486206,
+      "logits/rejected": -2.5850634574890137,
+      "logps/chosen": -0.9030648469924927,
+      "logps/rejected": -1.0534106492996216,
+      "loss": 0.9502,
+      "nll_loss": 0.9085375070571899,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.09030647575855255,
+      "rewards/margins": 0.015034586191177368,
+      "rewards/rejected": -0.10534106194972992,
+      "step": 180
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 10.63166001842217,
+      "learning_rate": 3.6273812505500587e-06,
+      "log_odds_chosen": 0.26296746730804443,
+      "log_odds_ratio": -0.6528322100639343,
+      "logits/chosen": -2.5235934257507324,
+      "logits/rejected": -2.509213924407959,
+      "logps/chosen": -0.842459499835968,
+      "logps/rejected": -1.0031650066375732,
+      "loss": 0.935,
+      "nll_loss": 0.840924084186554,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.0842459425330162,
+      "rewards/margins": 0.016070572659373283,
+      "rewards/rejected": -0.10031652450561523,
+      "step": 190
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 10.265720627085773,
+      "learning_rate": 3.5355339059327378e-06,
+      "log_odds_chosen": 0.20157551765441895,
+      "log_odds_ratio": -0.6874672174453735,
+      "logits/chosen": -2.5119268894195557,
+      "logits/rejected": -2.516270399093628,
+      "logps/chosen": -0.8817191123962402,
+      "logps/rejected": -1.0264997482299805,
+      "loss": 0.9827,
+      "nll_loss": 0.9070207476615906,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08817192167043686,
+      "rewards/margins": 0.01447806041687727,
+      "rewards/rejected": -0.10264997184276581,
+      "step": 200
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 10.139451950242082,
+      "learning_rate": 3.450327796711771e-06,
+      "log_odds_chosen": 0.22201482951641083,
+      "log_odds_ratio": -0.6832079887390137,
+      "logits/chosen": -2.438990831375122,
+      "logits/rejected": -2.4239249229431152,
+      "logps/chosen": -0.9554345011711121,
+      "logps/rejected": -1.1055123805999756,
+      "loss": 0.9961,
+      "nll_loss": 0.9768595695495605,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.09554344415664673,
+      "rewards/margins": 0.01500779576599598,
+      "rewards/rejected": -0.11055125296115875,
+      "step": 210
+    },
+    {
+      "epoch": 1.0,
+      "eval_log_odds_chosen": 0.3896147906780243,
+      "eval_log_odds_ratio": -0.6294437050819397,
+      "eval_logits/chosen": -2.359004259109497,
+      "eval_logits/rejected": -2.3542282581329346,
+      "eval_logps/chosen": -0.8311433792114258,
+      "eval_logps/rejected": -1.0547062158584595,
+      "eval_loss": 0.9198942184448242,
+      "eval_nll_loss": 0.8774265646934509,
+      "eval_rewards/accuracies": 0.6302083134651184,
+      "eval_rewards/chosen": -0.08311434835195541,
+      "eval_rewards/margins": 0.022356273606419563,
+      "eval_rewards/rejected": -0.10547062009572983,
+      "eval_runtime": 61.0411,
+      "eval_samples_per_second": 12.287,
+      "eval_steps_per_second": 0.393,
+      "step": 211
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 8.627109146931504,
+      "learning_rate": 3.3709993123162106e-06,
+      "log_odds_chosen": 1.4988970756530762,
+      "log_odds_ratio": -0.30856823921203613,
+      "logits/chosen": -2.3857474327087402,
+      "logits/rejected": -2.3801207542419434,
+      "logps/chosen": -0.45462799072265625,
+      "logps/rejected": -1.2372629642486572,
+      "loss": 0.564,
+      "nll_loss": 0.48940524458885193,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.045462802052497864,
+      "rewards/margins": 0.07826349139213562,
+      "rewards/rejected": -0.12372628599405289,
+      "step": 220
+    },
+    {
+      "epoch": 1.09,
+      "grad_norm": 10.837022751981742,
+      "learning_rate": 3.296902366978936e-06,
+      "log_odds_chosen": 1.769687294960022,
+      "log_odds_ratio": -0.22798781096935272,
+      "logits/chosen": -2.460228204727173,
+      "logits/rejected": -2.4452919960021973,
+      "logps/chosen": -0.48005446791648865,
+      "logps/rejected": -1.4746625423431396,
+      "loss": 0.5158,
+      "nll_loss": 0.5252029895782471,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.04800545051693916,
+      "rewards/margins": 0.09946081787347794,
+      "rewards/rejected": -0.1474662721157074,
+      "step": 230
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 9.220899180149999,
+      "learning_rate": 3.2274861218395142e-06,
+      "log_odds_chosen": 1.664380431175232,
+      "log_odds_ratio": -0.2567726671695709,
+      "logits/chosen": -2.4479331970214844,
+      "logits/rejected": -2.453798294067383,
+      "logps/chosen": -0.43317586183547974,
+      "logps/rejected": -1.2501039505004883,
+      "loss": 0.4887,
+      "nll_loss": 0.45028549432754517,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.04331757873296738,
+      "rewards/margins": 0.08169281482696533,
+      "rewards/rejected": -0.1250104010105133,
+      "step": 240
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 9.9378304724897,
+      "learning_rate": 3.1622776601683796e-06,
+      "log_odds_chosen": 1.5019989013671875,
+      "log_odds_ratio": -0.298888623714447,
+      "logits/chosen": -2.5074825286865234,
+      "logits/rejected": -2.4912190437316895,
+      "logps/chosen": -0.4875302314758301,
+      "logps/rejected": -1.2663142681121826,
+      "loss": 0.5504,
+      "nll_loss": 0.5222305655479431,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.04875302314758301,
+      "rewards/margins": 0.07787840068340302,
+      "rewards/rejected": -0.12663142383098602,
+      "step": 250
+    },
+    {
+      "epoch": 1.23,
+      "grad_norm": 10.42321736026305,
+      "learning_rate": 3.1008683647302113e-06,
+      "log_odds_chosen": 1.7215213775634766,
+      "log_odds_ratio": -0.2548612058162689,
+      "logits/chosen": -2.523898124694824,
+      "logits/rejected": -2.516082525253296,
+      "logps/chosen": -0.4069024920463562,
+      "logps/rejected": -1.2453140020370483,
+      "loss": 0.4878,
+      "nll_loss": 0.4438396394252777,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.04069024696946144,
+      "rewards/margins": 0.08384115993976593,
+      "rewards/rejected": -0.12453138828277588,
+      "step": 260
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 8.517195541285803,
+      "learning_rate": 3.0429030972509227e-06,
+      "log_odds_chosen": 1.4622563123703003,
+      "log_odds_ratio": -0.31491032242774963,
+      "logits/chosen": -2.421508312225342,
+      "logits/rejected": -2.4460442066192627,
+      "logps/chosen": -0.44591742753982544,
+      "logps/rejected": -1.127700924873352,
+      "loss": 0.5104,
+      "nll_loss": 0.48815250396728516,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.044591739773750305,
+      "rewards/margins": 0.06817835569381714,
+      "rewards/rejected": -0.11277009546756744,
+      "step": 270
+    },
+    {
+      "epoch": 1.33,
+      "grad_norm": 8.676525480706056,
+      "learning_rate": 2.988071523335984e-06,
+      "log_odds_chosen": 1.713667631149292,
+      "log_odds_ratio": -0.23984098434448242,
+      "logits/chosen": -2.433767795562744,
+      "logits/rejected": -2.4231998920440674,
+      "logps/chosen": -0.42581120133399963,
+      "logps/rejected": -1.254382848739624,
+      "loss": 0.4905,
+      "nll_loss": 0.4856826364994049,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.04258112236857414,
+      "rewards/margins": 0.0828571617603302,
+      "rewards/rejected": -0.12543828785419464,
+      "step": 280
+    },
+    {
+      "epoch": 1.37,
+      "grad_norm": 10.179660663431909,
+      "learning_rate": 2.9361010975735177e-06,
+      "log_odds_chosen": 1.6406803131103516,
+      "log_odds_ratio": -0.2432694435119629,
+      "logits/chosen": -2.40620493888855,
+      "logits/rejected": -2.462026357650757,
+      "logps/chosen": -0.43379467725753784,
+      "logps/rejected": -1.231093406677246,
+      "loss": 0.4952,
+      "nll_loss": 0.4767753481864929,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.043379463255405426,
+      "rewards/margins": 0.07972987741231918,
+      "rewards/rejected": -0.1231093555688858,
+      "step": 290
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 10.354476045311776,
+      "learning_rate": 2.8867513459481293e-06,
+      "log_odds_chosen": 1.8753684759140015,
+      "log_odds_ratio": -0.24006888270378113,
+      "logits/chosen": -2.4737935066223145,
+      "logits/rejected": -2.483096122741699,
+      "logps/chosen": -0.3926279544830322,
+      "logps/rejected": -1.2253657579421997,
+      "loss": 0.4883,
+      "nll_loss": 0.4013836979866028,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.0392627976834774,
+      "rewards/margins": 0.08327377587556839,
+      "rewards/rejected": -0.12253657728433609,
+      "step": 300
+    },
+    {
+      "epoch": 1.47,
+      "grad_norm": 9.015044311790971,
+      "learning_rate": 2.839809171235324e-06,
+      "log_odds_chosen": 1.8872253894805908,
+      "log_odds_ratio": -0.22469110786914825,
+      "logits/chosen": -2.400991678237915,
+      "logits/rejected": -2.397636890411377,
+      "logps/chosen": -0.40627264976501465,
+      "logps/rejected": -1.4336292743682861,
+      "loss": 0.5011,
+      "nll_loss": 0.43345755338668823,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.040627263486385345,
+      "rewards/margins": 0.10273567587137222,
+      "rewards/rejected": -0.14336295425891876,
+      "step": 310
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 9.539457700499925,
+      "learning_rate": 2.7950849718747376e-06,
+      "log_odds_chosen": 1.8418588638305664,
+      "log_odds_ratio": -0.20935162901878357,
+      "logits/chosen": -2.3977291584014893,
+      "logits/rejected": -2.3498239517211914,
+      "logps/chosen": -0.4429972767829895,
+      "logps/rejected": -1.4245294332504272,
+      "loss": 0.4827,
+      "nll_loss": 0.4764658510684967,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.04429972171783447,
+      "rewards/margins": 0.09815323352813721,
+      "rewards/rejected": -0.14245295524597168,
+      "step": 320
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 9.434503294939443,
+      "learning_rate": 2.752409412815902e-06,
+      "log_odds_chosen": 1.5989511013031006,
+      "log_odds_ratio": -0.26907485723495483,
+      "logits/chosen": -2.4358298778533936,
+      "logits/rejected": -2.4641692638397217,
+      "logps/chosen": -0.4505313038825989,
+      "logps/rejected": -1.2999070882797241,
+      "loss": 0.4685,
+      "nll_loss": 0.49505728483200073,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.04505313187837601,
+      "rewards/margins": 0.08493757992982864,
+      "rewards/rejected": -0.12999072670936584,
+      "step": 330
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 9.785466529429728,
+      "learning_rate": 2.711630722733202e-06,
+      "log_odds_chosen": 1.757354736328125,
+      "log_odds_ratio": -0.22594013810157776,
+      "logits/chosen": -2.386993885040283,
+      "logits/rejected": -2.3797829151153564,
+      "logps/chosen": -0.3836737275123596,
+      "logps/rejected": -1.226621389389038,
+      "loss": 0.5139,
+      "nll_loss": 0.4257877469062805,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0383673757314682,
+      "rewards/margins": 0.08429475873708725,
+      "rewards/rejected": -0.12266211211681366,
+      "step": 340
+    },
+    {
+      "epoch": 1.66,
+      "grad_norm": 9.783556860345605,
+      "learning_rate": 2.6726124191242444e-06,
+      "log_odds_chosen": 1.5875946283340454,
+      "log_odds_ratio": -0.282478392124176,
+      "logits/chosen": -2.4133667945861816,
+      "logits/rejected": -2.4031758308410645,
+      "logps/chosen": -0.47809138894081116,
+      "logps/rejected": -1.3111228942871094,
+      "loss": 0.4947,
+      "nll_loss": 0.46197766065597534,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.04780913516879082,
+      "rewards/margins": 0.08330314606428146,
+      "rewards/rejected": -0.13111227750778198,
+      "step": 350
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 10.952497066310066,
+      "learning_rate": 2.6352313834736496e-06,
+      "log_odds_chosen": 1.444085717201233,
+      "log_odds_ratio": -0.302844375371933,
+      "logits/chosen": -2.35420560836792,
+      "logits/rejected": -2.40644907951355,
+      "logps/chosen": -0.4726253151893616,
+      "logps/rejected": -1.181214451789856,
+      "loss": 0.5344,
+      "nll_loss": 0.5417958498001099,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.0472625270485878,
+      "rewards/margins": 0.0708589106798172,
+      "rewards/rejected": -0.1181214451789856,
+      "step": 360
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 9.337096555826308,
+      "learning_rate": 2.599376224550182e-06,
+      "log_odds_chosen": 1.5420761108398438,
+      "log_odds_ratio": -0.2706839144229889,
+      "logits/chosen": -2.393585681915283,
+      "logits/rejected": -2.3775479793548584,
+      "logps/chosen": -0.4294983744621277,
+      "logps/rejected": -1.177330493927002,
+      "loss": 0.489,
+      "nll_loss": 0.4465916156768799,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.04294983297586441,
+      "rewards/margins": 0.07478321343660355,
+      "rewards/rejected": -0.11773304641246796,
+      "step": 370
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 9.897099582842259,
+      "learning_rate": 2.564945880212886e-06,
+      "log_odds_chosen": 1.6586507558822632,
+      "log_odds_ratio": -0.2737709581851959,
+      "logits/chosen": -2.505715847015381,
+      "logits/rejected": -2.478332996368408,
+      "logps/chosen": -0.47623515129089355,
+      "logps/rejected": -1.3953008651733398,
+      "loss": 0.5232,
+      "nll_loss": 0.498232901096344,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.047623515129089355,
+      "rewards/margins": 0.0919065773487091,
+      "rewards/rejected": -0.13953009247779846,
+      "step": 380
+    },
+    {
+      "epoch": 1.85,
+      "grad_norm": 8.744712266716895,
+      "learning_rate": 2.5318484177091667e-06,
+      "log_odds_chosen": 1.4230409860610962,
+      "log_odds_ratio": -0.29352661967277527,
+      "logits/chosen": -2.455392360687256,
+      "logits/rejected": -2.4511191844940186,
+      "logps/chosen": -0.5203135013580322,
+      "logps/rejected": -1.2620891332626343,
+      "loss": 0.5658,
+      "nll_loss": 0.547887921333313,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.05203135684132576,
+      "rewards/margins": 0.0741775631904602,
+      "rewards/rejected": -0.12620893120765686,
+      "step": 390
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 9.30311341527825,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 1.7381293773651123,
+      "log_odds_ratio": -0.25197070837020874,
+      "logits/chosen": -2.512232780456543,
+      "logits/rejected": -2.5167810916900635,
+      "logps/chosen": -0.4353383183479309,
+      "logps/rejected": -1.34853196144104,
+      "loss": 0.5158,
+      "nll_loss": 0.4797208309173584,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.04353383183479309,
+      "rewards/margins": 0.09131935983896255,
+      "rewards/rejected": -0.13485319912433624,
+      "step": 400
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 9.272816070528886,
+      "learning_rate": 2.4693239916239746e-06,
+      "log_odds_chosen": 1.5647149085998535,
+      "log_odds_ratio": -0.29222574830055237,
+      "logits/chosen": -2.4848999977111816,
+      "logits/rejected": -2.4973931312561035,
+      "logps/chosen": -0.4486003816127777,
+      "logps/rejected": -1.2298743724822998,
+      "loss": 0.4991,
+      "nll_loss": 0.49500808119773865,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.04486004263162613,
+      "rewards/margins": 0.07812739908695221,
+      "rewards/rejected": -0.12298743426799774,
+      "step": 410
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 11.956370778331848,
+      "learning_rate": 2.4397501823713327e-06,
+      "log_odds_chosen": 1.8623790740966797,
+      "log_odds_ratio": -0.23835237324237823,
+      "logits/chosen": -2.491649866104126,
+      "logits/rejected": -2.4964215755462646,
+      "logps/chosen": -0.36661866307258606,
+      "logps/rejected": -1.2916817665100098,
+      "loss": 0.4999,
+      "nll_loss": 0.4245728850364685,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.03666186332702637,
+      "rewards/margins": 0.09250631183385849,
+      "rewards/rejected": -0.12916818261146545,
+      "step": 420
+    },
+    {
+      "epoch": 2.0,
+      "eval_log_odds_chosen": 0.4993869364261627,
+      "eval_log_odds_ratio": -0.6011503338813782,
+      "eval_logits/chosen": -2.51769757270813,
+      "eval_logits/rejected": -2.518415689468384,
+      "eval_logps/chosen": -0.8936966061592102,
+      "eval_logps/rejected": -1.183638334274292,
+      "eval_loss": 0.9812777638435364,
+      "eval_nll_loss": 0.940436840057373,
+      "eval_rewards/accuracies": 0.6614583134651184,
+      "eval_rewards/chosen": -0.08936966210603714,
+      "eval_rewards/margins": 0.028994165360927582,
+      "eval_rewards/rejected": -0.11836383491754532,
+      "eval_runtime": 60.9721,
+      "eval_samples_per_second": 12.301,
+      "eval_steps_per_second": 0.394,
+      "step": 422
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 11.137133583839702,
+      "learning_rate": 2.411214110852061e-06,
+      "log_odds_chosen": 2.8829193115234375,
+      "log_odds_ratio": -0.11160006374120712,
+      "logits/chosen": -2.481720209121704,
+      "logits/rejected": -2.5044140815734863,
+      "logps/chosen": -0.21169082820415497,
+      "logps/rejected": -1.3710792064666748,
+      "loss": 0.2521,
+      "nll_loss": 0.2048369199037552,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.021169083192944527,
+      "rewards/margins": 0.11593882739543915,
+      "rewards/rejected": -0.13710792362689972,
+      "step": 430
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 9.190185309499267,
+      "learning_rate": 2.3836564731139807e-06,
+      "log_odds_chosen": 3.475831985473633,
+      "log_odds_ratio": -0.0667722150683403,
+      "logits/chosen": -2.468839645385742,
+      "logits/rejected": -2.504293441772461,
+      "logps/chosen": -0.13927796483039856,
+      "logps/rejected": -1.5609562397003174,
+      "loss": 0.1889,
+      "nll_loss": 0.142722949385643,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.013927794992923737,
+      "rewards/margins": 0.1421678364276886,
+      "rewards/rejected": -0.15609565377235413,
+      "step": 440
+    },
+    {
+      "epoch": 2.13,
+      "grad_norm": 9.564387172693797,
+      "learning_rate": 2.357022603955159e-06,
+      "log_odds_chosen": 3.917105197906494,
+      "log_odds_ratio": -0.037336982786655426,
+      "logits/chosen": -2.5166149139404297,
+      "logits/rejected": -2.545985698699951,
+      "logps/chosen": -0.13963064551353455,
+      "logps/rejected": -1.9594411849975586,
+      "loss": 0.181,
+      "nll_loss": 0.15659931302070618,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.013963064178824425,
+      "rewards/margins": 0.18198105692863464,
+      "rewards/rejected": -0.19594410061836243,
+      "step": 450
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 9.849074304214747,
+      "learning_rate": 2.3312620206007847e-06,
+      "log_odds_chosen": 3.3254103660583496,
+      "log_odds_ratio": -0.06604982912540436,
+      "logits/chosen": -2.4885592460632324,
+      "logits/rejected": -2.5000545978546143,
+      "logps/chosen": -0.17423538863658905,
+      "logps/rejected": -1.5009019374847412,
+      "loss": 0.1952,
+      "nll_loss": 0.21155783534049988,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.017423538491129875,
+      "rewards/margins": 0.13266664743423462,
+      "rewards/rejected": -0.15009018778800964,
+      "step": 460
+    },
+    {
+      "epoch": 2.23,
+      "grad_norm": 9.877044500517252,
+      "learning_rate": 2.3063280200722128e-06,
+      "log_odds_chosen": 3.6100239753723145,
+      "log_odds_ratio": -0.05039358139038086,
+      "logits/chosen": -2.5267927646636963,
+      "logits/rejected": -2.5161995887756348,
+      "logps/chosen": -0.16191843152046204,
+      "logps/rejected": -1.8569713830947876,
+      "loss": 0.173,
+      "nll_loss": 0.16810880601406097,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.016191843897104263,
+      "rewards/margins": 0.1695052832365036,
+      "rewards/rejected": -0.18569712340831757,
+      "step": 470
+    },
+    {
+      "epoch": 2.27,
+      "grad_norm": 8.163874170175868,
+      "learning_rate": 2.2821773229381924e-06,
+      "log_odds_chosen": 3.612773895263672,
+      "log_odds_ratio": -0.05763618275523186,
+      "logits/chosen": -2.5359530448913574,
+      "logits/rejected": -2.539139747619629,
+      "logps/chosen": -0.17719587683677673,
+      "logps/rejected": -1.8914750814437866,
+      "loss": 0.1808,
+      "nll_loss": 0.17015354335308075,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.017719587311148643,
+      "rewards/margins": 0.171427920460701,
+      "rewards/rejected": -0.18914751708507538,
+      "step": 480
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 9.330097236244276,
+      "learning_rate": 2.2587697572631284e-06,
+      "log_odds_chosen": 3.656383514404297,
+      "log_odds_ratio": -0.05575472116470337,
+      "logits/chosen": -2.450357437133789,
+      "logits/rejected": -2.4894728660583496,
+      "logps/chosen": -0.12832140922546387,
+      "logps/rejected": -1.6764119863510132,
+      "loss": 0.1743,
+      "nll_loss": 0.1293896734714508,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.012832140550017357,
+      "rewards/margins": 0.15480905771255493,
+      "rewards/rejected": -0.16764120757579803,
+      "step": 490
+    },
+    {
+      "epoch": 2.37,
+      "grad_norm": 8.638065176436028,
+      "learning_rate": 2.23606797749979e-06,
+      "log_odds_chosen": 3.5653939247131348,
+      "log_odds_ratio": -0.04521235078573227,
+      "logits/chosen": -2.4729416370391846,
+      "logits/rejected": -2.4972083568573,
+      "logps/chosen": -0.14347168803215027,
+      "logps/rejected": -1.709639549255371,
+      "loss": 0.1687,
+      "nll_loss": 0.15304037928581238,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.014347168616950512,
+      "rewards/margins": 0.15661677718162537,
+      "rewards/rejected": -0.17096397280693054,
+      "step": 500
+    },
+    {
+      "epoch": 2.42,
+      "grad_norm": 9.821968133030161,
+      "learning_rate": 2.2140372138502386e-06,
+      "log_odds_chosen": 3.384936809539795,
+      "log_odds_ratio": -0.06072957068681717,
+      "logits/chosen": -2.477996349334717,
+      "logits/rejected": -2.482419729232788,
+      "logps/chosen": -0.17441074550151825,
+      "logps/rejected": -1.6888253688812256,
+      "loss": 0.191,
+      "nll_loss": 0.20637556910514832,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.017441075295209885,
+      "rewards/margins": 0.15144145488739014,
+      "rewards/rejected": -0.16888253390789032,
+      "step": 510
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 9.230367667532265,
+      "learning_rate": 2.1926450482675734e-06,
+      "log_odds_chosen": 3.698854446411133,
+      "log_odds_ratio": -0.04557035490870476,
+      "logits/chosen": -2.493439197540283,
+      "logits/rejected": -2.507626533508301,
+      "logps/chosen": -0.15794190764427185,
+      "logps/rejected": -1.836679458618164,
+      "loss": 0.1823,
+      "nll_loss": 0.1679120659828186,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.015794191509485245,
+      "rewards/margins": 0.16787376999855042,
+      "rewards/rejected": -0.18366795778274536,
+      "step": 520
+    },
+    {
+      "epoch": 2.51,
+      "grad_norm": 9.220963630868974,
+      "learning_rate": 2.1718612138153473e-06,
+      "log_odds_chosen": 3.2164981365203857,
+      "log_odds_ratio": -0.08045772463083267,
+      "logits/chosen": -2.4849820137023926,
+      "logits/rejected": -2.513134717941284,
+      "logps/chosen": -0.20204570889472961,
+      "logps/rejected": -1.6124742031097412,
+      "loss": 0.1955,
+      "nll_loss": 0.1980821043252945,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0202045738697052,
+      "rewards/margins": 0.14104285836219788,
+      "rewards/rejected": -0.16124743223190308,
+      "step": 530
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 10.33252022252946,
+      "learning_rate": 2.151657414559676e-06,
+      "log_odds_chosen": 3.355163097381592,
+      "log_odds_ratio": -0.05846261978149414,
+      "logits/chosen": -2.4703683853149414,
+      "logits/rejected": -2.4815986156463623,
+      "logps/chosen": -0.16165122389793396,
+      "logps/rejected": -1.595573902130127,
+      "loss": 0.1944,
+      "nll_loss": 0.1843605637550354,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.016165124252438545,
+      "rewards/margins": 0.14339227974414825,
+      "rewards/rejected": -0.15955740213394165,
+      "step": 540
+    },
+    {
+      "epoch": 2.61,
+      "grad_norm": 10.384732401123731,
+      "learning_rate": 2.132007163556104e-06,
+      "log_odds_chosen": 3.7364859580993652,
+      "log_odds_ratio": -0.050393521785736084,
+      "logits/chosen": -2.488259792327881,
+      "logits/rejected": -2.49784517288208,
+      "logps/chosen": -0.1458165943622589,
+      "logps/rejected": -1.7980148792266846,
+      "loss": 0.1656,
+      "nll_loss": 0.16633237898349762,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.014581659808754921,
+      "rewards/margins": 0.16521981358528137,
+      "rewards/rejected": -0.17980147898197174,
+      "step": 550
+    },
+    {
+      "epoch": 2.65,
+      "grad_norm": 8.88379907455603,
+      "learning_rate": 2.1128856368212917e-06,
+      "log_odds_chosen": 3.3721110820770264,
+      "log_odds_ratio": -0.05570857599377632,
+      "logits/chosen": -2.4560656547546387,
+      "logits/rejected": -2.459944725036621,
+      "logps/chosen": -0.14691922068595886,
+      "logps/rejected": -1.4988293647766113,
+      "loss": 0.1941,
+      "nll_loss": 0.16041871905326843,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.014691921882331371,
+      "rewards/margins": 0.13519100844860077,
+      "rewards/rejected": -0.1498829424381256,
+      "step": 560
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 8.564539697443587,
+      "learning_rate": 2.0942695414584777e-06,
+      "log_odds_chosen": 3.540961503982544,
+      "log_odds_ratio": -0.050423987209796906,
+      "logits/chosen": -2.4706387519836426,
+      "logits/rejected": -2.478842258453369,
+      "logps/chosen": -0.15486404299736023,
+      "logps/rejected": -1.6839189529418945,
+      "loss": 0.1827,
+      "nll_loss": 0.16179664433002472,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.015486404299736023,
+      "rewards/margins": 0.15290549397468567,
+      "rewards/rejected": -0.1683918982744217,
+      "step": 570
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 8.776123151000377,
+      "learning_rate": 2.0761369963434992e-06,
+      "log_odds_chosen": 3.2635269165039062,
+      "log_odds_ratio": -0.07641835510730743,
+      "logits/chosen": -2.49845552444458,
+      "logits/rejected": -2.5065455436706543,
+      "logps/chosen": -0.21721407771110535,
+      "logps/rejected": -1.628244161605835,
+      "loss": 0.2085,
+      "nll_loss": 0.26145899295806885,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.021721405908465385,
+      "rewards/margins": 0.14110301434993744,
+      "rewards/rejected": -0.16282442212104797,
+      "step": 580
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 8.360109402740795,
+      "learning_rate": 2.058467423981546e-06,
+      "log_odds_chosen": 3.5911190509796143,
+      "log_odds_ratio": -0.04746793955564499,
+      "logits/chosen": -2.4481544494628906,
+      "logits/rejected": -2.4686474800109863,
+      "logps/chosen": -0.18100793659687042,
+      "logps/rejected": -1.7877342700958252,
+      "loss": 0.1973,
+      "nll_loss": 0.19511719048023224,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.018100792542099953,
+      "rewards/margins": 0.1606726348400116,
+      "rewards/rejected": -0.178773432970047,
+      "step": 590
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 8.230484923418327,
+      "learning_rate": 2.0412414523193154e-06,
+      "log_odds_chosen": 3.52643084526062,
+      "log_odds_ratio": -0.05582552030682564,
+      "logits/chosen": -2.43485689163208,
+      "logits/rejected": -2.452052116394043,
+      "logps/chosen": -0.16544193029403687,
+      "logps/rejected": -1.6863620281219482,
+      "loss": 0.1863,
+      "nll_loss": 0.17863860726356506,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.016544191166758537,
+      "rewards/margins": 0.15209202468395233,
+      "rewards/rejected": -0.16863620281219482,
+      "step": 600
+    },
+    {
+      "epoch": 2.89,
+      "grad_norm": 9.657714694141937,
+      "learning_rate": 2.0244408254472904e-06,
+      "log_odds_chosen": 3.441495418548584,
+      "log_odds_ratio": -0.050580836832523346,
+      "logits/chosen": -2.3969621658325195,
+      "logits/rejected": -2.4254117012023926,
+      "logps/chosen": -0.1414327472448349,
+      "logps/rejected": -1.5730435848236084,
+      "loss": 0.1825,
+      "nll_loss": 0.16234278678894043,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01414327509701252,
+      "rewards/margins": 0.1431610882282257,
+      "rewards/rejected": -0.15730436146259308,
+      "step": 610
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 9.418203083185244,
+      "learning_rate": 2.0080483222562476e-06,
+      "log_odds_chosen": 3.510298252105713,
+      "log_odds_ratio": -0.04852527379989624,
+      "logits/chosen": -2.365495443344116,
+      "logits/rejected": -2.3816208839416504,
+      "logps/chosen": -0.14439718425273895,
+      "logps/rejected": -1.5629642009735107,
+      "loss": 0.1816,
+      "nll_loss": 0.1693393439054489,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.014439716935157776,
+      "rewards/margins": 0.14185671508312225,
+      "rewards/rejected": -0.15629643201828003,
+      "step": 620
+    },
+    {
+      "epoch": 2.99,
+      "grad_norm": 9.905181462463698,
+      "learning_rate": 1.9920476822239895e-06,
+      "log_odds_chosen": 3.6201272010803223,
+      "log_odds_ratio": -0.04453429579734802,
+      "logits/chosen": -2.415280818939209,
+      "logits/rejected": -2.403134822845459,
+      "logps/chosen": -0.1919378936290741,
+      "logps/rejected": -1.9056262969970703,
+      "loss": 0.1899,
+      "nll_loss": 0.1791400909423828,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01919379085302353,
+      "rewards/margins": 0.17136886715888977,
+      "rewards/rejected": -0.1905626356601715,
+      "step": 630
+    },
+    {
+      "epoch": 3.0,
+      "eval_log_odds_chosen": 0.539490818977356,
+      "eval_log_odds_ratio": -0.6108235716819763,
+      "eval_logits/chosen": -2.4178202152252197,
+      "eval_logits/rejected": -2.420111894607544,
+      "eval_logps/chosen": -1.1129909753799438,
+      "eval_logps/rejected": -1.4606622457504272,
+      "eval_loss": 1.2152478694915771,
+      "eval_nll_loss": 1.1710882186889648,
+      "eval_rewards/accuracies": 0.65625,
+      "eval_rewards/chosen": -0.11129910498857498,
+      "eval_rewards/margins": 0.03476712107658386,
+      "eval_rewards/rejected": -0.14606623351573944,
+      "eval_runtime": 60.9563,
+      "eval_samples_per_second": 12.304,
+      "eval_steps_per_second": 0.394,
+      "step": 633
+    },
+    {
+      "epoch": 3.0,
+      "step": 633,
+      "total_flos": 0.0,
+      "train_loss": 0.5526923979816467,
+      "train_runtime": 6310.4087,
+      "train_samples_per_second": 3.209,
+      "train_steps_per_second": 0.1
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 633,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acaf9c7a2cb692271963d3f4fb8c92c54b23a09fd264a214d4e80f2eba74fd39
+size 6456