Model save

README.md

@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2897.3179
-- Rewards/chosen: -0.6441
-- Rewards/rejected: -0.5754
-- Rewards/accuracies: 0.4239
-- Rewards/margins: -0.0687
-- Rewards/safe Rewards: -0.6417
-- Rewards/unsafe Rewards: -0.6386
-- Logps/rejected: -150.0089
-- Logps/chosen: -194.8470
-- Logits/rejected: -1.5786
-- Logits/chosen: -1.8526
+- Loss: 2401.7881
+- Rewards/chosen: -0.5946
+- Rewards/rejected: -0.5347
+- Rewards/accuracies: 0.4253
+- Rewards/margins: -0.0598
+- Rewards/safe Rewards: -0.5954
+- Rewards/unsafe Rewards: -0.5909
+- Logps/rejected: -145.8263
+- Logps/chosen: -190.3238
+- Logits/rejected: -1.5723
+- Logits/chosen: -1.8515
 
 ## Model description
 
@@ -49,10 +49,10 @@ The following hyperparameters were used during training:
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 4
-- gradient_accumulation_steps: 4
+- num_devices: 2
+- gradient_accumulation_steps: 8
 - total_train_batch_size: 32
-- total_eval_batch_size: 32
+- total_eval_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -62,9 +62,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 4952.3219     | 0.27  | 500  | 3031.1638       | -0.6585        | -0.5935          | 0.4215             | -0.0650         | -0.6552              | -0.6514                | -151.8219      | -196.2862    | -1.6812         | -1.9554       |
-| 4670.3188     | 0.54  | 1000 | 2931.8032       | -0.6547        | -0.5940          | 0.4353             | -0.0607         | -0.6514              | -0.6477                | -151.8720      | -195.9055    | -1.5084         | -1.7952       |
-| 4368.5492     | 0.81  | 1500 | 2899.6733       | -0.6504        | -0.5839          | 0.4268             | -0.0665         | -0.6481              | -0.6452                | -150.8582      | -195.4787    | -1.5552         | -1.8340       |
+| 4371.9453     | 0.27  | 500  | 2511.3528       | -0.5579        | -0.5108          | 0.4292             | -0.0471         | -0.5588              | -0.5535                | -143.4282      | -186.6566    | -1.5714         | -1.8674       |
+| 4141.4633     | 0.54  | 1000 | 2428.8169       | -0.6030        | -0.5520          | 0.4347             | -0.0511         | -0.6022              | -0.5984                | -147.5502      | -191.1693    | -1.5770         | -1.8581       |
+| 3953.5484     | 0.81  | 1500 | 2403.6704       | -0.5995        | -0.5427          | 0.4308             | -0.0569         | -0.6003              | -0.5961                | -146.6184      | -190.8177    | -1.5526         | -1.8366       |
 
 
 ### Framework versions
@@ -72,4 +72,4 @@ The following hyperparameters were used during training:
 - Transformers 4.36.2
 - Pytorch 2.1.2
 - Datasets 2.14.6
-- Tokenizers 0.15.0
+- Tokenizers 0.15.2

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -1.8526086807250977,
-    "eval_logits/rejected": -1.5786068439483643,
-    "eval_logps/chosen": -194.84695434570312,
-    "eval_logps/rejected": -150.0089111328125,
-    "eval_loss": 2897.31787109375,
-    "eval_rewards/accuracies": 0.4238867461681366,
-    "eval_rewards/chosen": -0.644080638885498,
-    "eval_rewards/margins": -0.06870394200086594,
-    "eval_rewards/rejected": -0.5753766894340515,
-    "eval_rewards/safe_rewards": -0.6417209506034851,
-    "eval_rewards/unsafe_rewards": -0.6385891437530518,
-    "eval_runtime": 1137.2097,
+    "eval_logits/chosen": -1.8515318632125854,
+    "eval_logits/rejected": -1.5723339319229126,
+    "eval_logps/chosen": -190.3237762451172,
+    "eval_logps/rejected": -145.82626342773438,
+    "eval_loss": 2401.7880859375,
+    "eval_rewards/accuracies": 0.4253388047218323,
+    "eval_rewards/chosen": -0.5945791006088257,
+    "eval_rewards/margins": -0.05983531475067139,
+    "eval_rewards/rejected": -0.5347437262535095,
+    "eval_rewards/safe_rewards": -0.5953946113586426,
+    "eval_rewards/unsafe_rewards": -0.5908674597740173,
+    "eval_runtime": 1810.1244,
     "eval_samples": 33044,
-    "eval_samples_per_second": 29.057,
-    "eval_steps_per_second": 0.908,
-    "train_loss": 4921.10222895923,
-    "train_runtime": 23079.3417,
+    "eval_samples_per_second": 18.255,
+    "eval_steps_per_second": 1.141,
+    "train_loss": 4385.191492448197,
+    "train_runtime": 39558.3676,
     "train_samples": 59478,
-    "train_samples_per_second": 2.577,
-    "train_steps_per_second": 0.081
+    "train_samples_per_second": 1.504,
+    "train_steps_per_second": 0.047
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -1.8526086807250977,
-    "eval_logits/rejected": -1.5786068439483643,
-    "eval_logps/chosen": -194.84695434570312,
-    "eval_logps/rejected": -150.0089111328125,
-    "eval_loss": 2897.31787109375,
-    "eval_rewards/accuracies": 0.4238867461681366,
-    "eval_rewards/chosen": -0.644080638885498,
-    "eval_rewards/margins": -0.06870394200086594,
-    "eval_rewards/rejected": -0.5753766894340515,
-    "eval_rewards/safe_rewards": -0.6417209506034851,
-    "eval_rewards/unsafe_rewards": -0.6385891437530518,
-    "eval_runtime": 1137.2097,
+    "eval_logits/chosen": -1.8515318632125854,
+    "eval_logits/rejected": -1.5723339319229126,
+    "eval_logps/chosen": -190.3237762451172,
+    "eval_logps/rejected": -145.82626342773438,
+    "eval_loss": 2401.7880859375,
+    "eval_rewards/accuracies": 0.4253388047218323,
+    "eval_rewards/chosen": -0.5945791006088257,
+    "eval_rewards/margins": -0.05983531475067139,
+    "eval_rewards/rejected": -0.5347437262535095,
+    "eval_rewards/safe_rewards": -0.5953946113586426,
+    "eval_rewards/unsafe_rewards": -0.5908674597740173,
+    "eval_runtime": 1810.1244,
     "eval_samples": 33044,
-    "eval_samples_per_second": 29.057,
-    "eval_steps_per_second": 0.908
+    "eval_samples_per_second": 18.255,
+    "eval_steps_per_second": 1.141
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20432f8815518e6c964d71ac826f847ac8e950fb5b92850b8341440459907ce2
+oid sha256:ccfaf958107cc66a3bba1c7da20f3bedd1358ea008a9cd6dd35123de0b327540
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3938ebe0667ca2758ffb46354beca6da6b14fead06b688f37e085ca48171e2de
+oid sha256:b4693d6fcbfdbb9cf069b3eeb5a4f562244a24ab4dacb3e1c3f0cee4d0d594f7
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb3c6d62e5e682e54493eac6f2ecc97e4f409e240a1ef8ae126617c8927def02
+oid sha256:01bb8843109bb068880c8cd5bab1de26aebf315ec4e127ad98ac90b6c145b0d3
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 4921.10222895923,
-    "train_runtime": 23079.3417,
+    "train_loss": 4385.191492448197,
+    "train_runtime": 39558.3676,
     "train_samples": 59478,
-    "train_samples_per_second": 2.577,
-    "train_steps_per_second": 0.081
+    "train_samples_per_second": 1.504,
+    "train_steps_per_second": 0.047
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e099ffa447833afe7e5d004b504ac1f4e68c81ba736a9aa279f79302e6ccfe0b
+oid sha256:fc8b1f505de6f23f53fe2c4fb0c58cd3c24df28746c7d0f66368d6136d47681e
 size 6200