Model save

README.md

@@ -2,16 +2,10 @@
 license: other
 base_model: HuggingFaceH4/zephyr-7b-gemma-sft-v0.1
 tags:
-- alignment-handbook
-- trl
-- dpo
-- generated_from_trainer
 - trl
 - dpo
 - alignment-handbook
 - generated_from_trainer
-datasets:
-- argilla/dpo-mix-7k
 model-index:
 - name: zephyr-7b-gemma-dpo
   results: []
@@ -22,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-gemma-dpo
 
-This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on the argilla/dpo-mix-7k dataset.
+This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4653
-- Rewards/chosen: -4.1128
-- Rewards/rejected: -5.6919
-- Rewards/accuracies: 0.7292
-- Rewards/margins: 1.5791
-- Logps/rejected: -475.9087
-- Logps/chosen: -445.9241
-- Logits/rejected: 91.1136
-- Logits/chosen: 96.7260
+- Loss: 0.4643
+- Rewards/chosen: -3.5909
+- Rewards/rejected: -5.3391
+- Rewards/accuracies: 0.75
+- Rewards/margins: 1.7481
+- Logps/rejected: -515.7638
+- Logps/chosen: -428.1683
+- Logits/rejected: 94.0722
+- Logits/chosen: 91.3541
 
 ## Model description
 
@@ -56,10 +50,10 @@ The following hyperparameters were used during training:
 - eval_batch_size: 4
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 8
-- gradient_accumulation_steps: 8
+- num_devices: 4
+- gradient_accumulation_steps: 16
 - total_train_batch_size: 128
-- total_eval_batch_size: 32
+- total_eval_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -69,7 +63,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.1585        | 1.8957 | 100  | 0.4673          | -4.1293        | -5.7148          | 0.6979             | 1.5855          | -476.3664      | -446.2537    | 91.1323         | 96.7315       |
+| 0.1578        | 1.8957 | 100  | 0.4643          | -3.5909        | -5.3391          | 0.75               | 1.7481          | -515.7638      | -428.1683    | 94.0722         | 91.3541       |
 
 
 ### Framework versions

all_results.json

@@ -1,22 +1,22 @@
 {
     "epoch": 1.971563981042654,
-    "eval_logits/chosen": 96.72599029541016,
-    "eval_logits/rejected": 91.11358642578125,
-    "eval_logps/chosen": -445.924072265625,
-    "eval_logps/rejected": -475.90869140625,
-    "eval_loss": 0.4652560353279114,
-    "eval_rewards/accuracies": 0.7291666865348816,
-    "eval_rewards/chosen": -4.112792491912842,
-    "eval_rewards/margins": 1.5790935754776,
-    "eval_rewards/rejected": -5.691885471343994,
-    "eval_runtime": 119.6024,
+    "eval_logits/chosen": 96.71578216552734,
+    "eval_logits/rejected": 90.98221588134766,
+    "eval_logps/chosen": -423.6227722167969,
+    "eval_logps/rejected": -453.7782287597656,
+    "eval_loss": 0.468290776014328,
+    "eval_rewards/accuracies": 0.7708333134651184,
+    "eval_rewards/chosen": -3.0221338272094727,
+    "eval_rewards/margins": 1.6591955423355103,
+    "eval_rewards/rejected": -4.681329727172852,
+    "eval_runtime": 58.6185,
     "eval_samples": 750,
-    "eval_samples_per_second": 6.271,
-    "eval_steps_per_second": 0.201,
+    "eval_samples_per_second": 12.795,
+    "eval_steps_per_second": 0.409,
     "total_flos": 0.0,
-    "train_loss": 0.39153398688022906,
-    "train_runtime": 2311.0387,
+    "train_loss": 0.3883641087091886,
+    "train_runtime": 2802.2739,
     "train_samples": 6750,
-    "train_samples_per_second": 5.842,
-    "train_steps_per_second": 0.045
+    "train_samples_per_second": 4.818,
+    "train_steps_per_second": 0.037
 }

config.json

@@ -24,6 +24,6 @@
   "rope_theta": 10000.0,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.1",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 256000
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.971563981042654,
-    "eval_logits/chosen": 96.72599029541016,
-    "eval_logits/rejected": 91.11358642578125,
-    "eval_logps/chosen": -445.924072265625,
-    "eval_logps/rejected": -475.90869140625,
-    "eval_loss": 0.4652560353279114,
-    "eval_rewards/accuracies": 0.7291666865348816,
-    "eval_rewards/chosen": -4.112792491912842,
-    "eval_rewards/margins": 1.5790935754776,
-    "eval_rewards/rejected": -5.691885471343994,
-    "eval_runtime": 119.6024,
+    "eval_logits/chosen": 96.71578216552734,
+    "eval_logits/rejected": 90.98221588134766,
+    "eval_logps/chosen": -423.6227722167969,
+    "eval_logps/rejected": -453.7782287597656,
+    "eval_loss": 0.468290776014328,
+    "eval_rewards/accuracies": 0.7708333134651184,
+    "eval_rewards/chosen": -3.0221338272094727,
+    "eval_rewards/margins": 1.6591955423355103,
+    "eval_rewards/rejected": -4.681329727172852,
+    "eval_runtime": 58.6185,
     "eval_samples": 750,
-    "eval_samples_per_second": 6.271,
-    "eval_steps_per_second": 0.201
+    "eval_samples_per_second": 12.795,
+    "eval_steps_per_second": 0.409
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b26faf7dcff7b7ca7bbf4ebc9d54968ab56cd1bbe5b3f4059d0ec34b7b1ccdd2
+oid sha256:861ddf07decf97620b031a6e15e48a651e1034c23bd959c3ff531de2cb3fc3ef
 size 4995496656

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:98a41992ae6f80c80eaa24e7d8dbecab5d07c2802028c109568fe70565b4c6d8
+oid sha256:08603a5d632cb46ee7b055c6a36a5a07d2166b085310ae9aca36b53732222289
 size 4982953168

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d1d7ab5de3f2e26234060bf0c99e343d3a84489614f455b267bd22f059fc862
+oid sha256:5bf49dcc8070ec107b57ff3c2256c6c28740b70300f7e1cb5c38b020fdd478da
 size 4982953200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07f5983e107d05b629942a14afa7af7fe7e3836b05bc872e472789542c0f95b6
+oid sha256:c3202396afc49b66318136fd06f7a9728e8cb9b624a3d5aaa731944d527748aa
 size 2113988336

runs/Apr29_18-08-25_gcp002/events.out.tfevents.1714414192.gcp002.13104.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7df9aecf4657b82643e8f0eca97479f3cc310ea6d9af0d5dc3a4377609afee8
+size 13441

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.971563981042654,
     "total_flos": 0.0,
-    "train_loss": 0.39153398688022906,
-    "train_runtime": 2311.0387,
+    "train_loss": 0.3883641087091886,
+    "train_runtime": 2802.2739,
     "train_samples": 6750,
-    "train_samples_per_second": 5.842,
-    "train_steps_per_second": 0.045
+    "train_samples_per_second": 4.818,
+    "train_steps_per_second": 0.037
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.018957345971563982,
-      "grad_norm": 132.15360444004384,
+      "grad_norm": 133.64062565621384,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": 117.53560638427734,
-      "logits/rejected": 126.8960952758789,
-      "logps/chosen": -335.40118408203125,
-      "logps/rejected": -439.16552734375,
+      "logits/chosen": 119.0696792602539,
+      "logits/rejected": 120.28123474121094,
+      "logps/chosen": -394.1268310546875,
+      "logps/rejected": -419.3145446777344,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,178 +25,178 @@
     },
     {
       "epoch": 0.1895734597156398,
-      "grad_norm": 132.3674027987073,
+      "grad_norm": 130.60842697521545,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": 135.01699829101562,
-      "logits/rejected": 138.37664794921875,
-      "logps/chosen": -396.05718994140625,
-      "logps/rejected": -439.1203918457031,
-      "loss": 0.7127,
-      "rewards/accuracies": 0.4583333432674408,
-      "rewards/chosen": -0.0030322629027068615,
-      "rewards/margins": -0.013390823267400265,
-      "rewards/rejected": 0.010358559899032116,
+      "logits/chosen": 133.6595001220703,
+      "logits/rejected": 136.7303466796875,
+      "logps/chosen": -410.0771484375,
+      "logps/rejected": -445.1907653808594,
+      "loss": 0.7019,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.020121444016695023,
+      "rewards/margins": 0.041466910392045975,
+      "rewards/rejected": -0.021345460787415504,
       "step": 10
     },
     {
       "epoch": 0.3791469194312796,
-      "grad_norm": 131.21733523095625,
+      "grad_norm": 127.29787487076526,
       "learning_rate": 4.885348141000122e-07,
-      "logits/chosen": 121.60444641113281,
-      "logits/rejected": 125.29842376708984,
-      "logps/chosen": -370.2664489746094,
-      "logps/rejected": -422.78851318359375,
-      "loss": 0.6459,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.10727670043706894,
-      "rewards/margins": 0.247134730219841,
-      "rewards/rejected": -0.13985800743103027,
+      "logits/chosen": 122.2022476196289,
+      "logits/rejected": 128.57586669921875,
+      "logps/chosen": -357.1582336425781,
+      "logps/rejected": -416.08087158203125,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": 0.25420495867729187,
+      "rewards/margins": 0.4108888506889343,
+      "rewards/rejected": -0.15668384730815887,
       "step": 20
     },
     {
       "epoch": 0.5687203791469194,
-      "grad_norm": 117.90232463642135,
+      "grad_norm": 110.05011163607695,
       "learning_rate": 4.5025027361734613e-07,
-      "logits/chosen": 142.974853515625,
-      "logits/rejected": 136.52386474609375,
-      "logps/chosen": -424.7781677246094,
-      "logps/rejected": -469.64813232421875,
-      "loss": 0.5746,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.6156466007232666,
-      "rewards/margins": 0.8666501045227051,
-      "rewards/rejected": -2.4822967052459717,
+      "logits/chosen": 121.9586181640625,
+      "logits/rejected": 125.2878646850586,
+      "logps/chosen": -387.713134765625,
+      "logps/rejected": -442.55206298828125,
+      "loss": 0.5698,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -1.2848999500274658,
+      "rewards/margins": 0.9555079340934753,
+      "rewards/rejected": -2.240407943725586,
       "step": 30
     },
     {
       "epoch": 0.7582938388625592,
-      "grad_norm": 104.91283452119073,
+      "grad_norm": 111.08969508053838,
       "learning_rate": 3.893311157806091e-07,
-      "logits/chosen": 126.9936752319336,
-      "logits/rejected": 115.53365325927734,
-      "logps/chosen": -399.81353759765625,
-      "logps/rejected": -426.99853515625,
-      "loss": 0.5456,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.2809689044952393,
-      "rewards/margins": 1.1751956939697266,
-      "rewards/rejected": -3.456164598464966,
+      "logits/chosen": 121.52265930175781,
+      "logits/rejected": 119.2688980102539,
+      "logps/chosen": -402.15716552734375,
+      "logps/rejected": -444.649169921875,
+      "loss": 0.5496,
+      "rewards/accuracies": 0.7593749761581421,
+      "rewards/chosen": -2.0494799613952637,
+      "rewards/margins": 1.070623517036438,
+      "rewards/rejected": -3.120103359222412,
       "step": 40
     },
     {
       "epoch": 0.9478672985781991,
-      "grad_norm": 123.57780236639618,
+      "grad_norm": 122.82358054602282,
       "learning_rate": 3.126631330646801e-07,
-      "logits/chosen": 142.1190643310547,
-      "logits/rejected": 146.2515411376953,
-      "logps/chosen": -456.97979736328125,
-      "logps/rejected": -540.1392822265625,
-      "loss": 0.489,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.891798734664917,
-      "rewards/margins": 1.2988468408584595,
-      "rewards/rejected": -3.190645456314087,
+      "logits/chosen": 128.3933868408203,
+      "logits/rejected": 133.44308471679688,
+      "logps/chosen": -431.0421447753906,
+      "logps/rejected": -497.99420166015625,
+      "loss": 0.4959,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": -2.2058186531066895,
+      "rewards/margins": 1.2984471321105957,
+      "rewards/rejected": -3.504265546798706,
       "step": 50
     },
     {
       "epoch": 1.1374407582938388,
-      "grad_norm": 67.1680971334559,
+      "grad_norm": 65.56687198861316,
       "learning_rate": 2.2891223348923882e-07,
-      "logits/chosen": 133.56114196777344,
-      "logits/rejected": 137.20738220214844,
-      "logps/chosen": -449.55303955078125,
-      "logps/rejected": -534.8367919921875,
-      "loss": 0.3117,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -2.5773684978485107,
-      "rewards/margins": 2.346193790435791,
-      "rewards/rejected": -4.923562049865723,
+      "logits/chosen": 124.70857238769531,
+      "logits/rejected": 126.91219329833984,
+      "logps/chosen": -420.981201171875,
+      "logps/rejected": -505.5345153808594,
+      "loss": 0.307,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -2.4674336910247803,
+      "rewards/margins": 2.305318832397461,
+      "rewards/rejected": -4.772752285003662,
       "step": 60
     },
     {
       "epoch": 1.3270142180094786,
-      "grad_norm": 48.54475300946312,
+      "grad_norm": 56.106028687537446,
       "learning_rate": 1.4754491880085317e-07,
-      "logits/chosen": 125.71492004394531,
-      "logits/rejected": 127.68719482421875,
-      "logps/chosen": -426.90228271484375,
-      "logps/rejected": -528.0679321289062,
-      "loss": 0.195,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": -2.9879212379455566,
-      "rewards/margins": 2.7397806644439697,
-      "rewards/rejected": -5.727701663970947,
+      "logits/chosen": 121.775146484375,
+      "logits/rejected": 125.95316314697266,
+      "logps/chosen": -425.7054138183594,
+      "logps/rejected": -518.8656005859375,
+      "loss": 0.1907,
+      "rewards/accuracies": 0.940625011920929,
+      "rewards/chosen": -2.623661518096924,
+      "rewards/margins": 2.869920253753662,
+      "rewards/rejected": -5.493582248687744,
       "step": 70
     },
     {
       "epoch": 1.5165876777251186,
-      "grad_norm": 55.371866892062,
+      "grad_norm": 50.43661058282089,
       "learning_rate": 7.775827023107834e-08,
-      "logits/chosen": 111.7248306274414,
-      "logits/rejected": 128.3420867919922,
-      "logps/chosen": -427.53106689453125,
-      "logps/rejected": -546.7640991210938,
-      "loss": 0.1651,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -3.693999767303467,
-      "rewards/margins": 2.9823195934295654,
-      "rewards/rejected": -6.676319122314453,
+      "logits/chosen": 114.5962142944336,
+      "logits/rejected": 126.1790771484375,
+      "logps/chosen": -426.8082580566406,
+      "logps/rejected": -527.3065185546875,
+      "loss": 0.1761,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -2.979158401489258,
+      "rewards/margins": 3.0644469261169434,
+      "rewards/rejected": -6.043605804443359,
       "step": 80
     },
     {
       "epoch": 1.7061611374407581,
-      "grad_norm": 40.99464664899818,
+      "grad_norm": 45.81843583580765,
       "learning_rate": 2.7440387297912122e-08,
-      "logits/chosen": 110.8941879272461,
-      "logits/rejected": 123.70848083496094,
-      "logps/chosen": -457.2183532714844,
-      "logps/rejected": -575.8634033203125,
-      "loss": 0.1557,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -3.5393664836883545,
-      "rewards/margins": 3.369715929031372,
-      "rewards/rejected": -6.909082889556885,
+      "logits/chosen": 117.46388244628906,
+      "logits/rejected": 123.80489349365234,
+      "logps/chosen": -449.65399169921875,
+      "logps/rejected": -544.6094970703125,
+      "loss": 0.1515,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -2.8478360176086426,
+      "rewards/margins": 3.247156858444214,
+      "rewards/rejected": -6.0949931144714355,
       "step": 90
     },
     {
       "epoch": 1.8957345971563981,
-      "grad_norm": 45.385328063823785,
+      "grad_norm": 42.75820426735574,
       "learning_rate": 2.27878296044029e-09,
-      "logits/chosen": 117.1551284790039,
-      "logits/rejected": 117.0487060546875,
-      "logps/chosen": -446.9934997558594,
-      "logps/rejected": -541.2728881835938,
-      "loss": 0.1585,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -3.250919818878174,
-      "rewards/margins": 2.9745240211486816,
-      "rewards/rejected": -6.2254438400268555,
+      "logits/chosen": 114.7729721069336,
+      "logits/rejected": 119.34477233886719,
+      "logps/chosen": -437.2296447753906,
+      "logps/rejected": -523.9191284179688,
+      "loss": 0.1578,
+      "rewards/accuracies": 0.9593750238418579,
+      "rewards/chosen": -2.8138155937194824,
+      "rewards/margins": 3.170293092727661,
+      "rewards/rejected": -5.984108924865723,
       "step": 100
     },
     {
       "epoch": 1.8957345971563981,
-      "eval_logits/chosen": 96.73149871826172,
-      "eval_logits/rejected": 91.1323013305664,
-      "eval_logps/chosen": -446.253662109375,
-      "eval_logps/rejected": -476.3663635253906,
-      "eval_loss": 0.46732592582702637,
-      "eval_rewards/accuracies": 0.6979166865348816,
-      "eval_rewards/chosen": -4.1292724609375,
-      "eval_rewards/margins": 1.5854991674423218,
-      "eval_rewards/rejected": -5.714771270751953,
-      "eval_runtime": 120.4793,
-      "eval_samples_per_second": 6.225,
-      "eval_steps_per_second": 0.199,
+      "eval_logits/chosen": 91.35408782958984,
+      "eval_logits/rejected": 94.07221221923828,
+      "eval_logps/chosen": -428.1683349609375,
+      "eval_logps/rejected": -515.7637939453125,
+      "eval_loss": 0.4643263816833496,
+      "eval_rewards/accuracies": 0.75,
+      "eval_rewards/chosen": -3.5909416675567627,
+      "eval_rewards/margins": 1.7481167316436768,
+      "eval_rewards/rejected": -5.339057922363281,
+      "eval_runtime": 88.3612,
+      "eval_samples_per_second": 8.488,
+      "eval_steps_per_second": 0.532,
       "step": 100
     },
     {
       "epoch": 1.971563981042654,
       "step": 104,
       "total_flos": 0.0,
-      "train_loss": 0.39153398688022906,
-      "train_runtime": 2311.0387,
-      "train_samples_per_second": 5.842,
-      "train_steps_per_second": 0.045
+      "train_loss": 0.3883641087091886,
+      "train_runtime": 2802.2739,
+      "train_samples_per_second": 4.818,
+      "train_steps_per_second": 0.037
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79ee78a4306adfc04ffa07fc0ca8acbb9d3417b9d7c9f4adaf815a8d83ea6a24
+oid sha256:b835231394e8e7d484d57fdd04805c7ac65d3f2e0c869e656ccf783b2d023691
 size 6264