Model save

README.md

@@ -0,0 +1,73 @@
+---
+license: apache-2.0
+base_model: tiiuae/falcon-7b
+tags:
+- generated_from_trainer
+model-index:
+- name: falcon-7b-dpo-lora
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# falcon-7b-dpo-lora
+
+This model is a fine-tuned version of [tiiuae/falcon-7b](https://huggingface.co/tiiuae/falcon-7b) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6909
+- Rewards/chosen: 0.0159
+- Rewards/rejected: 0.0096
+- Rewards/accuracies: 0.3175
+- Rewards/margins: 0.0063
+- Logps/rejected: -88.4353
+- Logps/chosen: -109.9771
+- Logits/rejected: -14.9384
+- Logits/chosen: -14.8499
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 32
+- total_train_batch_size: 512
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6927        | 1.0   | 121  | 0.6929          | 0.0035         | 0.0015           | 0.2857             | 0.0020          | -88.5168       | -110.1007    | -14.9346        | -14.8473      |
+| 0.6915        | 2.0   | 242  | 0.6917          | 0.0116         | 0.0071           | 0.3056             | 0.0045          | -88.4605       | -110.0199    | -14.9351        | -14.8469      |
+| 0.6913        | 3.0   | 363  | 0.6909          | 0.0159         | 0.0096           | 0.3175             | 0.0063          | -88.4353       | -109.9771    | -14.9384        | -14.8499      |
+
+
+### Framework versions
+
+- Transformers 4.35.0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.14.1

adapter_config.json

@@ -0,0 +1,25 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "tiiuae/falcon-7b",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "dense",
+    "query_key_value",
+    "dense_h_to_4h",
+    "dense_4h_to_h"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16fda22ae48812ffcb9872506da16398e686133d835f80ef6baf8abc8a97f07b
+size 522227376

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 3.0,
+    "eval_logits/chosen": -14.8499116897583,
+    "eval_logits/rejected": -14.938364028930664,
+    "eval_logps/chosen": -109.97705078125,
+    "eval_logps/rejected": -88.43526458740234,
+    "eval_loss": 0.6909456849098206,
+    "eval_rewards/accuracies": 0.3174603283405304,
+    "eval_rewards/chosen": 0.015893306583166122,
+    "eval_rewards/margins": 0.006259124726057053,
+    "eval_rewards/rejected": 0.00963417999446392,
+    "eval_runtime": 227.6793,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 8.784,
+    "eval_steps_per_second": 0.277,
+    "train_loss": 0.692123620142293,
+    "train_runtime": 31275.9572,
+    "train_samples": 61966,
+    "train_samples_per_second": 5.944,
+    "train_steps_per_second": 0.012
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 3.0,
+    "eval_logits/chosen": -14.8499116897583,
+    "eval_logits/rejected": -14.938364028930664,
+    "eval_logps/chosen": -109.97705078125,
+    "eval_logps/rejected": -88.43526458740234,
+    "eval_loss": 0.6909456849098206,
+    "eval_rewards/accuracies": 0.3174603283405304,
+    "eval_rewards/chosen": 0.015893306583166122,
+    "eval_rewards/margins": 0.006259124726057053,
+    "eval_rewards/rejected": 0.00963417999446392,
+    "eval_runtime": 227.6793,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 8.784,
+    "eval_steps_per_second": 0.277
+}

runs/Dec19_01-00-45_ip-172-16-14-202.ec2.internal/events.out.tfevents.1702947856.ip-172-16-14-202.ec2.internal.114331.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1e5cc2365c3f367566a6b386369ff2aa8794ca42ee26834f4750c276db23757
+size 5045

runs/Dec19_01-09-14_ip-172-16-14-202.ec2.internal/events.out.tfevents.1702948337.ip-172-16-14-202.ec2.internal.122402.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af402a7627f5995dde748e15d6bfa41f9897a532bd42aef2a518d5a06c73fca5
+size 30920

runs/Dec19_01-09-14_ip-172-16-14-202.ec2.internal/events.out.tfevents.1702979838.ip-172-16-14-202.ec2.internal.122402.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6574ffedb1b41cda2d6015be5bc8900b52e7a389f5abed37f19204dcfdd7564
+size 828

special_tokens_map.json

@@ -0,0 +1,29 @@
+{
+  "additional_special_tokens": [
+    ">>TITLE<<",
+    ">>ABSTRACT<<",
+    ">>INTRODUCTION<<",
+    ">>SUMMARY<<",
+    ">>COMMENT<<",
+    ">>ANSWER<<",
+    ">>QUESTION<<",
+    ">>DOMAIN<<",
+    ">>PREFIX<<",
+    ">>SUFFIX<<",
+    ">>MIDDLE<<"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,124 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": ">>TITLE<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": ">>ABSTRACT<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": ">>INTRODUCTION<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": ">>SUMMARY<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": ">>COMMENT<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": ">>ANSWER<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": ">>QUESTION<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": ">>DOMAIN<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": ">>PREFIX<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": ">>SUFFIX<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": ">>MIDDLE<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    ">>TITLE<<",
+    ">>ABSTRACT<<",
+    ">>INTRODUCTION<<",
+    ">>SUMMARY<<",
+    ">>COMMENT<<",
+    ">>ANSWER<<",
+    ">>QUESTION<<",
+    ">>DOMAIN<<",
+    ">>PREFIX<<",
+    ">>SUFFIX<<",
+    ">>MIDDLE<<"
+  ],
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 2048,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.692123620142293,
+    "train_runtime": 31275.9572,
+    "train_samples": 61966,
+    "train_samples_per_second": 5.944,
+    "train_steps_per_second": 0.012
+}

trainer_state.json

@@ -0,0 +1,594 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9992254066615027,
+  "eval_steps": 100,
+  "global_step": 363,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "learning_rate": 1.3513513513513514e-08,
+      "logits/chosen": -15.172547340393066,
+      "logits/rejected": -15.251230239868164,
+      "logps/chosen": -103.97029113769531,
+      "logps/rejected": -79.94886016845703,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.3513513513513515e-07,
+      "logits/chosen": -15.22812557220459,
+      "logits/rejected": -15.30654525756836,
+      "logps/chosen": -123.74452209472656,
+      "logps/rejected": -93.8215560913086,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.2309027761220932,
+      "rewards/chosen": 0.0003177134203724563,
+      "rewards/margins": 0.00015111881657503545,
+      "rewards/rejected": 0.000166594676556997,
+      "step": 10
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 2.702702702702703e-07,
+      "logits/chosen": -15.262018203735352,
+      "logits/rejected": -15.291465759277344,
+      "logps/chosen": -123.9498291015625,
+      "logps/rejected": -96.19405364990234,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.29374998807907104,
+      "rewards/chosen": 0.0009166754898615181,
+      "rewards/margins": 0.0013821485918015242,
+      "rewards/rejected": -0.00046547286910936236,
+      "step": 20
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.054054054054054e-07,
+      "logits/chosen": -15.272542953491211,
+      "logits/rejected": -15.305673599243164,
+      "logps/chosen": -127.54842376708984,
+      "logps/rejected": -105.3931884765625,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.2796874940395355,
+      "rewards/chosen": 0.00042984067113138735,
+      "rewards/margins": -0.00048730484559200704,
+      "rewards/rejected": 0.0009171456913463771,
+      "step": 30
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.95398773006135e-07,
+      "logits/chosen": -15.2285737991333,
+      "logits/rejected": -15.296609878540039,
+      "logps/chosen": -124.9699935913086,
+      "logps/rejected": -97.49568176269531,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.26249998807907104,
+      "rewards/chosen": 0.0005645907949656248,
+      "rewards/margins": 0.00030518617131747305,
+      "rewards/rejected": 0.00025940462364815176,
+      "step": 40
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 4.800613496932515e-07,
+      "logits/chosen": -15.285560607910156,
+      "logits/rejected": -15.345190048217773,
+      "logps/chosen": -129.31256103515625,
+      "logps/rejected": -99.52166748046875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.2796874940395355,
+      "rewards/chosen": 0.00027664678054861724,
+      "rewards/margins": 5.752297511207871e-05,
+      "rewards/rejected": 0.00021912386000622064,
+      "step": 50
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 4.647239263803681e-07,
+      "logits/chosen": -15.253962516784668,
+      "logits/rejected": -15.33386516571045,
+      "logps/chosen": -119.763427734375,
+      "logps/rejected": -102.88087463378906,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.2796874940395355,
+      "rewards/chosen": 0.000503324088640511,
+      "rewards/margins": 8.147591142915189e-05,
+      "rewards/rejected": 0.0004218483227305114,
+      "step": 60
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.4938650306748465e-07,
+      "logits/chosen": -15.288751602172852,
+      "logits/rejected": -15.326438903808594,
+      "logps/chosen": -127.03987121582031,
+      "logps/rejected": -96.76243591308594,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.2890625,
+      "rewards/chosen": 0.0017284611240029335,
+      "rewards/margins": 0.0012586858356371522,
+      "rewards/rejected": 0.0004697750264313072,
+      "step": 70
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 4.340490797546012e-07,
+      "logits/chosen": -15.288309097290039,
+      "logits/rejected": -15.354669570922852,
+      "logps/chosen": -129.9634552001953,
+      "logps/rejected": -101.72119140625,
+      "loss": 0.693,
+      "rewards/accuracies": 0.28125,
+      "rewards/chosen": 0.0008187053608708084,
+      "rewards/margins": 0.0004409000393934548,
+      "rewards/rejected": 0.0003778053796850145,
+      "step": 80
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 4.187116564417178e-07,
+      "logits/chosen": -15.274989128112793,
+      "logits/rejected": -15.232473373413086,
+      "logps/chosen": -121.2137451171875,
+      "logps/rejected": -113.5948486328125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.296875,
+      "rewards/chosen": 0.001987897325307131,
+      "rewards/margins": 0.0013456062879413366,
+      "rewards/rejected": 0.0006422909209504724,
+      "step": 90
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.0337423312883434e-07,
+      "logits/chosen": -15.281481742858887,
+      "logits/rejected": -15.288961410522461,
+      "logps/chosen": -113.20597839355469,
+      "logps/rejected": -97.28117370605469,
+      "loss": 0.693,
+      "rewards/accuracies": 0.2874999940395355,
+      "rewards/chosen": 0.0011146818287670612,
+      "rewards/margins": 0.00026254478143528104,
+      "rewards/rejected": 0.000852137163747102,
+      "step": 100
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 3.880368098159509e-07,
+      "logits/chosen": -15.232090950012207,
+      "logits/rejected": -15.308767318725586,
+      "logps/chosen": -131.0157012939453,
+      "logps/rejected": -104.84590911865234,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.28125,
+      "rewards/chosen": 0.0027348899748176336,
+      "rewards/margins": 0.0014507302548736334,
+      "rewards/rejected": 0.0012841597199440002,
+      "step": 110
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.7269938650306747e-07,
+      "logits/chosen": -15.231793403625488,
+      "logits/rejected": -15.243997573852539,
+      "logps/chosen": -120.3092269897461,
+      "logps/rejected": -101.40711975097656,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.26875001192092896,
+      "rewards/chosen": 0.0026273976545780897,
+      "rewards/margins": 0.0005600537406280637,
+      "rewards/rejected": 0.002067344030365348,
+      "step": 120
+    },
+    {
+      "epoch": 1.0,
+      "eval_logits/chosen": -14.84730052947998,
+      "eval_logits/rejected": -14.93464183807373,
+      "eval_logps/chosen": -110.10073852539062,
+      "eval_logps/rejected": -88.51683044433594,
+      "eval_loss": 0.6929101943969727,
+      "eval_rewards/accuracies": 0.2857142984867096,
+      "eval_rewards/chosen": 0.0035249004140496254,
+      "eval_rewards/margins": 0.0020474623888731003,
+      "eval_rewards/rejected": 0.001477438141591847,
+      "eval_runtime": 230.1482,
+      "eval_samples_per_second": 8.69,
+      "eval_steps_per_second": 0.274,
+      "step": 121
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 3.5736196319018404e-07,
+      "logits/chosen": -15.243385314941406,
+      "logits/rejected": -15.28660774230957,
+      "logps/chosen": -117.83067321777344,
+      "logps/rejected": -105.49190521240234,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.3031249940395355,
+      "rewards/chosen": 0.0028103243093937635,
+      "rewards/margins": 0.0009912283858284354,
+      "rewards/rejected": 0.0018190961563959718,
+      "step": 130
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 3.420245398773006e-07,
+      "logits/chosen": -15.298578262329102,
+      "logits/rejected": -15.308378219604492,
+      "logps/chosen": -117.3888168334961,
+      "logps/rejected": -95.77253723144531,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.28125,
+      "rewards/chosen": 0.003832989139482379,
+      "rewards/margins": 0.0002880497486330569,
+      "rewards/rejected": 0.0035449396818876266,
+      "step": 140
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 3.2668711656441716e-07,
+      "logits/chosen": -15.269906997680664,
+      "logits/rejected": -15.330175399780273,
+      "logps/chosen": -121.6659927368164,
+      "logps/rejected": -98.5518569946289,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.28593748807907104,
+      "rewards/chosen": 0.004216608125716448,
+      "rewards/margins": 0.0014314096188172698,
+      "rewards/rejected": 0.0027851988561451435,
+      "step": 150
+    },
+    {
+      "epoch": 1.32,
+      "learning_rate": 3.1134969325153373e-07,
+      "logits/chosen": -15.28375244140625,
+      "logits/rejected": -15.307037353515625,
+      "logps/chosen": -117.98995208740234,
+      "logps/rejected": -101.3696517944336,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.2874999940395355,
+      "rewards/chosen": 0.004791011102497578,
+      "rewards/margins": 0.0008280017646029592,
+      "rewards/rejected": 0.00396300945430994,
+      "step": 160
+    },
+    {
+      "epoch": 1.4,
+      "learning_rate": 2.960122699386503e-07,
+      "logits/chosen": -15.264017105102539,
+      "logits/rejected": -15.2572603225708,
+      "logps/chosen": -125.29273986816406,
+      "logps/rejected": -112.38765716552734,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": 0.006073893513530493,
+      "rewards/margins": 0.0020425966940820217,
+      "rewards/rejected": 0.004031297750771046,
+      "step": 170
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 2.8067484662576686e-07,
+      "logits/chosen": -15.275349617004395,
+      "logits/rejected": -15.314315795898438,
+      "logps/chosen": -120.6910400390625,
+      "logps/rejected": -92.7549057006836,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.3343749940395355,
+      "rewards/chosen": 0.006351888179779053,
+      "rewards/margins": 0.002958633005619049,
+      "rewards/rejected": 0.0033932551741600037,
+      "step": 180
+    },
+    {
+      "epoch": 1.57,
+      "learning_rate": 2.653374233128834e-07,
+      "logits/chosen": -15.27843189239502,
+      "logits/rejected": -15.3081693649292,
+      "logps/chosen": -115.19669342041016,
+      "logps/rejected": -92.90242004394531,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.3218750059604645,
+      "rewards/chosen": 0.0064470237120985985,
+      "rewards/margins": 0.0023960156831890345,
+      "rewards/rejected": 0.004051008261740208,
+      "step": 190
+    },
+    {
+      "epoch": 1.65,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -15.239352226257324,
+      "logits/rejected": -15.272363662719727,
+      "logps/chosen": -121.6275634765625,
+      "logps/rejected": -93.08324432373047,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.2734375,
+      "rewards/chosen": 0.0069998181425035,
+      "rewards/margins": 0.0022552732843905687,
+      "rewards/rejected": 0.004744545556604862,
+      "step": 200
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 2.3466257668711655e-07,
+      "logits/chosen": -15.199145317077637,
+      "logits/rejected": -15.267333984375,
+      "logps/chosen": -124.52889251708984,
+      "logps/rejected": -97.74082946777344,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.2593750059604645,
+      "rewards/chosen": 0.0072901747189462185,
+      "rewards/margins": 0.0013283558655530214,
+      "rewards/rejected": 0.005961819086223841,
+      "step": 210
+    },
+    {
+      "epoch": 1.82,
+      "learning_rate": 2.1932515337423312e-07,
+      "logits/chosen": -15.27015495300293,
+      "logits/rejected": -15.279447555541992,
+      "logps/chosen": -118.91484069824219,
+      "logps/rejected": -90.32121276855469,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.30156248807907104,
+      "rewards/chosen": 0.00907520018517971,
+      "rewards/margins": 0.0032898869831115007,
+      "rewards/rejected": 0.005785313434898853,
+      "step": 220
+    },
+    {
+      "epoch": 1.9,
+      "learning_rate": 2.0398773006134968e-07,
+      "logits/chosen": -15.252248764038086,
+      "logits/rejected": -15.287099838256836,
+      "logps/chosen": -117.89067077636719,
+      "logps/rejected": -100.09212493896484,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.3187499940395355,
+      "rewards/chosen": 0.00985044240951538,
+      "rewards/margins": 0.004119374789297581,
+      "rewards/rejected": 0.0057310680858790874,
+      "step": 230
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 1.8865030674846625e-07,
+      "logits/chosen": -15.214279174804688,
+      "logits/rejected": -15.315710067749023,
+      "logps/chosen": -121.98006439208984,
+      "logps/rejected": -98.08609008789062,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.3125,
+      "rewards/chosen": 0.010013025254011154,
+      "rewards/margins": 0.0034062224440276623,
+      "rewards/rejected": 0.00660680141299963,
+      "step": 240
+    },
+    {
+      "epoch": 2.0,
+      "eval_logits/chosen": -14.846867561340332,
+      "eval_logits/rejected": -14.935086250305176,
+      "eval_logps/chosen": -110.01992797851562,
+      "eval_logps/rejected": -88.46048736572266,
+      "eval_loss": 0.6916756629943848,
+      "eval_rewards/accuracies": 0.3055555522441864,
+      "eval_rewards/chosen": 0.011606148444116116,
+      "eval_rewards/margins": 0.00449267216026783,
+      "eval_rewards/rejected": 0.007113476749509573,
+      "eval_runtime": 229.7499,
+      "eval_samples_per_second": 8.705,
+      "eval_steps_per_second": 0.274,
+      "step": 242
+    },
+    {
+      "epoch": 2.07,
+      "learning_rate": 1.733128834355828e-07,
+      "logits/chosen": -15.271708488464355,
+      "logits/rejected": -15.325927734375,
+      "logps/chosen": -120.98309326171875,
+      "logps/rejected": -108.2812728881836,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.296875,
+      "rewards/chosen": 0.009962974116206169,
+      "rewards/margins": 0.0022991253063082695,
+      "rewards/rejected": 0.007663849741220474,
+      "step": 250
+    },
+    {
+      "epoch": 2.15,
+      "learning_rate": 1.5797546012269938e-07,
+      "logits/chosen": -15.266347885131836,
+      "logits/rejected": -15.258427619934082,
+      "logps/chosen": -123.05375671386719,
+      "logps/rejected": -98.55216979980469,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.2984375059604645,
+      "rewards/chosen": 0.011418156325817108,
+      "rewards/margins": 0.0035811502020806074,
+      "rewards/rejected": 0.007837006822228432,
+      "step": 260
+    },
+    {
+      "epoch": 2.23,
+      "learning_rate": 1.4263803680981594e-07,
+      "logits/chosen": -15.247785568237305,
+      "logits/rejected": -15.258626937866211,
+      "logps/chosen": -115.1404037475586,
+      "logps/rejected": -93.46723175048828,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.3375000059604645,
+      "rewards/chosen": 0.011196712031960487,
+      "rewards/margins": 0.004130281042307615,
+      "rewards/rejected": 0.007066431455314159,
+      "step": 270
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 1.273006134969325e-07,
+      "logits/chosen": -15.278572082519531,
+      "logits/rejected": -15.311563491821289,
+      "logps/chosen": -128.5188446044922,
+      "logps/rejected": -100.33396911621094,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.3046875,
+      "rewards/chosen": 0.012510362081229687,
+      "rewards/margins": 0.004159414209425449,
+      "rewards/rejected": 0.008350946940481663,
+      "step": 280
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 1.1196319018404908e-07,
+      "logits/chosen": -15.250155448913574,
+      "logits/rejected": -15.272921562194824,
+      "logps/chosen": -119.62606048583984,
+      "logps/rejected": -103.21083068847656,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.296875,
+      "rewards/chosen": 0.013957071118056774,
+      "rewards/margins": 0.004067812114953995,
+      "rewards/rejected": 0.00988925900310278,
+      "step": 290
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 9.662576687116564e-08,
+      "logits/chosen": -15.27168083190918,
+      "logits/rejected": -15.364572525024414,
+      "logps/chosen": -119.32635498046875,
+      "logps/rejected": -98.04741668701172,
+      "loss": 0.691,
+      "rewards/accuracies": 0.29218751192092896,
+      "rewards/chosen": 0.012033398263156414,
+      "rewards/margins": 0.0029196850955486298,
+      "rewards/rejected": 0.009113713167607784,
+      "step": 300
+    },
+    {
+      "epoch": 2.56,
+      "learning_rate": 8.12883435582822e-08,
+      "logits/chosen": -15.267038345336914,
+      "logits/rejected": -15.33867359161377,
+      "logps/chosen": -125.45201110839844,
+      "logps/rejected": -98.39042663574219,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.32499998807907104,
+      "rewards/chosen": 0.013260738924145699,
+      "rewards/margins": 0.004113315604627132,
+      "rewards/rejected": 0.00914742425084114,
+      "step": 310
+    },
+    {
+      "epoch": 2.64,
+      "learning_rate": 6.595092024539877e-08,
+      "logits/chosen": -15.279388427734375,
+      "logits/rejected": -15.33612060546875,
+      "logps/chosen": -123.69743347167969,
+      "logps/rejected": -101.3545150756836,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.32343751192092896,
+      "rewards/chosen": 0.013454603031277657,
+      "rewards/margins": 0.00448086392134428,
+      "rewards/rejected": 0.008973738178610802,
+      "step": 320
+    },
+    {
+      "epoch": 2.73,
+      "learning_rate": 5.061349693251534e-08,
+      "logits/chosen": -15.248504638671875,
+      "logits/rejected": -15.330392837524414,
+      "logps/chosen": -123.80655670166016,
+      "logps/rejected": -98.1585922241211,
+      "loss": 0.691,
+      "rewards/accuracies": 0.3187499940395355,
+      "rewards/chosen": 0.014067019335925579,
+      "rewards/margins": 0.005277669057250023,
+      "rewards/rejected": 0.008789352141320705,
+      "step": 330
+    },
+    {
+      "epoch": 2.81,
+      "learning_rate": 3.5276073619631896e-08,
+      "logits/chosen": -15.255121231079102,
+      "logits/rejected": -15.33825397491455,
+      "logps/chosen": -123.04536437988281,
+      "logps/rejected": -98.64664459228516,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": 0.014080156572163105,
+      "rewards/margins": 0.005763668101280928,
+      "rewards/rejected": 0.008316488936543465,
+      "step": 340
+    },
+    {
+      "epoch": 2.89,
+      "learning_rate": 1.9938650306748464e-08,
+      "logits/chosen": -15.204339981079102,
+      "logits/rejected": -15.2611722946167,
+      "logps/chosen": -127.34835052490234,
+      "logps/rejected": -111.3030014038086,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.33281248807907104,
+      "rewards/chosen": 0.015759991481900215,
+      "rewards/margins": 0.0036518643610179424,
+      "rewards/rejected": 0.012108126655220985,
+      "step": 350
+    },
+    {
+      "epoch": 2.97,
+      "learning_rate": 4.601226993865031e-09,
+      "logits/chosen": -15.213580131530762,
+      "logits/rejected": -15.218714714050293,
+      "logps/chosen": -105.95635986328125,
+      "logps/rejected": -87.81764221191406,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.2828125059604645,
+      "rewards/chosen": 0.012317867949604988,
+      "rewards/margins": 0.002315809251740575,
+      "rewards/rejected": 0.010002057068049908,
+      "step": 360
+    },
+    {
+      "epoch": 3.0,
+      "eval_logits/chosen": -14.8499116897583,
+      "eval_logits/rejected": -14.938364028930664,
+      "eval_logps/chosen": -109.97705078125,
+      "eval_logps/rejected": -88.43526458740234,
+      "eval_loss": 0.6909456849098206,
+      "eval_rewards/accuracies": 0.3174603283405304,
+      "eval_rewards/chosen": 0.015893306583166122,
+      "eval_rewards/margins": 0.006259124726057053,
+      "eval_rewards/rejected": 0.00963417999446392,
+      "eval_runtime": 227.7514,
+      "eval_samples_per_second": 8.782,
+      "eval_steps_per_second": 0.277,
+      "step": 363
+    },
+    {
+      "epoch": 3.0,
+      "step": 363,
+      "total_flos": 0.0,
+      "train_loss": 0.692123620142293,
+      "train_runtime": 31275.9572,
+      "train_samples_per_second": 5.944,
+      "train_steps_per_second": 0.012
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 363,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9d6abdf1534129e22e0e5ec0f15163fe17cfd28a5efc40ad9681b68547d064e
+size 4728