Model save

README.md

@@ -2,14 +2,9 @@
 license: mit
 base_model: ShenaoZ/0.000001_ablation_iter_1
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - dpo
 - generated_from_trainer
-datasets:
-- updated
-- original
 model-index:
 - name: 0.000001_ablation_iter_2
   results: []
@@ -20,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # 0.000001_ablation_iter_2
 
-This model is a fine-tuned version of [ShenaoZ/0.000001_ablation_iter_1](https://huggingface.co/ShenaoZ/0.000001_ablation_iter_1) on the updated and the original datasets.
+This model is a fine-tuned version of [ShenaoZ/0.000001_ablation_iter_1](https://huggingface.co/ShenaoZ/0.000001_ablation_iter_1) on the None dataset.
 
 ## Model description
 
@@ -39,7 +34,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-07
+- learning_rate: 5e-08
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6196039487730782,
-    "train_runtime": 4544.1519,
+    "train_loss": 0.6778795974059675,
+    "train_runtime": 4527.8424,
     "train_samples": 20378,
-    "train_samples_per_second": 4.484,
+    "train_samples_per_second": 4.501,
     "train_steps_per_second": 0.035
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:324c2f09643fcd1822c070c557fa2dd3f92afb6ef4e9c831cc3ae8315eb16f52
+oid sha256:f172b4876355bffd1c99db9ed0ff55ea0b13e97905ac5957345f5dcba4933e1c
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:06f021d09b06ec33d5740c6d7cf0f2b97239cb7b72910b8d6bc6e298fd916719
+oid sha256:88e4b007b183a5060399f7b1583faf3fac7306acd85af5e3627b920e9113d496
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c56d3610590d3ccb0fe70161afe4d49d33a1188ceca389387eb0395e5967ca3
+oid sha256:5823f950165df81548655dac76782683dc3c954240aea8cff2e4c281f65fba1d
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6196039487730782,
-    "train_runtime": 4544.1519,
+    "train_loss": 0.6778795974059675,
+    "train_runtime": 4527.8424,
     "train_samples": 20378,
-    "train_samples_per_second": 4.484,
+    "train_samples_per_second": 4.501,
     "train_steps_per_second": 0.035
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.01,
-      "learning_rate": 3.125e-08,
+      "learning_rate": 3.125e-09,
       "logits/chosen": -2.6344056129455566,
       "logits/rejected": -2.5906338691711426,
       "logps/chosen": -158.64126586914062,
@@ -26,251 +26,251 @@
     },
     {
       "epoch": 0.06,
-      "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.6918654441833496,
-      "logits/rejected": -2.6960322856903076,
-      "logps/chosen": -233.523193359375,
-      "logps/pi_response": -119.82765197753906,
+      "learning_rate": 3.125e-08,
+      "logits/chosen": -2.6937406063079834,
+      "logits/rejected": -2.697803258895874,
+      "logps/chosen": -232.31423950195312,
+      "logps/pi_response": -118.80592346191406,
       "logps/ref_response": -118.89913940429688,
-      "logps/rejected": -252.52438354492188,
-      "loss": 0.6919,
+      "logps/rejected": -251.14076232910156,
+      "loss": 0.6932,
       "rewards/accuracies": 0.4166666567325592,
-      "rewards/chosen": -0.011804760433733463,
-      "rewards/margins": 0.0010884494986385107,
-      "rewards/rejected": -0.012893210165202618,
+      "rewards/chosen": 0.00028485539951361716,
+      "rewards/margins": -0.0006583214271813631,
+      "rewards/rejected": 0.0009431770886294544,
       "step": 10
     },
     {
       "epoch": 0.13,
-      "learning_rate": 4.990353313429303e-07,
-      "logits/chosen": -2.6992347240448,
-      "logits/rejected": -2.6707592010498047,
-      "logps/chosen": -249.53662109375,
-      "logps/pi_response": -145.63510131835938,
+      "learning_rate": 4.990353313429303e-08,
+      "logits/chosen": -2.7444348335266113,
+      "logits/rejected": -2.718367338180542,
+      "logps/chosen": -232.554443359375,
+      "logps/pi_response": -131.306884765625,
       "logps/ref_response": -131.12376403808594,
-      "logps/rejected": -276.0727233886719,
-      "loss": 0.6793,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.17235596477985382,
-      "rewards/margins": 0.051038146018981934,
-      "rewards/rejected": -0.22339412569999695,
+      "logps/rejected": -254.0548553466797,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.00253383070230484,
+      "rewards/margins": 0.0006815333035774529,
+      "rewards/rejected": -0.003215363947674632,
       "step": 20
     },
     {
       "epoch": 0.19,
-      "learning_rate": 4.882681251368548e-07,
-      "logits/chosen": -2.5087711811065674,
-      "logits/rejected": -2.4484305381774902,
-      "logps/chosen": -241.63394165039062,
-      "logps/pi_response": -130.4356689453125,
+      "learning_rate": 4.8826812513685484e-08,
+      "logits/chosen": -2.6866040229797363,
+      "logits/rejected": -2.6289615631103516,
+      "logps/chosen": -227.64889526367188,
+      "logps/pi_response": -114.9122085571289,
       "logps/ref_response": -113.8835220336914,
-      "logps/rejected": -247.1987762451172,
-      "loss": 0.6667,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.15320873260498047,
-      "rewards/margins": 0.11732661724090576,
-      "rewards/rejected": -0.27053534984588623,
+      "logps/rejected": -221.958251953125,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.013358505442738533,
+      "rewards/margins": 0.004771661013364792,
+      "rewards/rejected": -0.018130164593458176,
       "step": 30
     },
     {
       "epoch": 0.25,
-      "learning_rate": 4.6604720940421207e-07,
-      "logits/chosen": -2.3236470222473145,
-      "logits/rejected": -2.3019630908966064,
-      "logps/chosen": -246.91586303710938,
-      "logps/pi_response": -157.4510955810547,
+      "learning_rate": 4.6604720940421204e-08,
+      "logits/chosen": -2.658876895904541,
+      "logits/rejected": -2.6415140628814697,
+      "logps/chosen": -206.0329132080078,
+      "logps/pi_response": -113.687744140625,
       "logps/ref_response": -110.53662109375,
-      "logps/rejected": -321.6424560546875,
-      "loss": 0.6426,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.4462863802909851,
-      "rewards/margins": 0.18701747059822083,
-      "rewards/rejected": -0.6333038806915283,
+      "logps/rejected": -263.42254638671875,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.03745666891336441,
+      "rewards/margins": 0.013648083433508873,
+      "rewards/rejected": -0.051104746758937836,
       "step": 40
     },
     {
       "epoch": 0.31,
-      "learning_rate": 4.3344075855595097e-07,
-      "logits/chosen": -2.238029718399048,
-      "logits/rejected": -2.208010673522949,
-      "logps/chosen": -281.1034240722656,
-      "logps/pi_response": -178.50894165039062,
+      "learning_rate": 4.3344075855595095e-08,
+      "logits/chosen": -2.6783342361450195,
+      "logits/rejected": -2.6550586223602295,
+      "logps/chosen": -232.83877563476562,
+      "logps/pi_response": -125.37564849853516,
       "logps/ref_response": -120.1254653930664,
-      "logps/rejected": -306.91387939453125,
-      "loss": 0.632,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.5530503392219543,
-      "rewards/margins": 0.14163774251937866,
-      "rewards/rejected": -0.694688081741333,
+      "logps/rejected": -245.05154418945312,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.0704035758972168,
+      "rewards/margins": 0.005661136005073786,
+      "rewards/rejected": -0.07606470584869385,
       "step": 50
     },
     {
       "epoch": 0.38,
-      "learning_rate": 3.920161866827889e-07,
-      "logits/chosen": -2.135256767272949,
-      "logits/rejected": -2.1016762256622314,
-      "logps/chosen": -281.59112548828125,
-      "logps/pi_response": -191.9367218017578,
+      "learning_rate": 3.920161866827889e-08,
+      "logits/chosen": -2.647156238555908,
+      "logits/rejected": -2.6198782920837402,
+      "logps/chosen": -222.61935424804688,
+      "logps/pi_response": -114.81854248046875,
       "logps/ref_response": -107.5267333984375,
-      "logps/rejected": -324.2335510253906,
-      "loss": 0.6292,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.6793836951255798,
-      "rewards/margins": 0.3076621890068054,
-      "rewards/rejected": -0.9870458841323853,
+      "logps/rejected": -237.662109375,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.08966623246669769,
+      "rewards/margins": 0.0316656231880188,
+      "rewards/rejected": -0.12133185565471649,
       "step": 60
     },
     {
       "epoch": 0.44,
-      "learning_rate": 3.4376480090239047e-07,
-      "logits/chosen": -2.1213016510009766,
-      "logits/rejected": -2.0694327354431152,
-      "logps/chosen": -281.278564453125,
-      "logps/pi_response": -191.5978546142578,
+      "learning_rate": 3.437648009023905e-08,
+      "logits/chosen": -2.644777536392212,
+      "logits/rejected": -2.615166664123535,
+      "logps/chosen": -226.83267211914062,
+      "logps/pi_response": -119.3505859375,
       "logps/ref_response": -112.31196594238281,
-      "logps/rejected": -297.12188720703125,
-      "loss": 0.6027,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.6307815313339233,
-      "rewards/margins": 0.3501848578453064,
-      "rewards/rejected": -0.9809663891792297,
+      "logps/rejected": -212.1949920654297,
+      "loss": 0.6752,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.08632274717092514,
+      "rewards/margins": 0.04537486657500267,
+      "rewards/rejected": -0.1316976249217987,
       "step": 70
     },
     {
       "epoch": 0.5,
-      "learning_rate": 2.910060778827554e-07,
-      "logits/chosen": -2.125455856323242,
-      "logits/rejected": -2.0651440620422363,
-      "logps/chosen": -259.68060302734375,
-      "logps/pi_response": -181.3156280517578,
+      "learning_rate": 2.9100607788275543e-08,
+      "logits/chosen": -2.640345573425293,
+      "logits/rejected": -2.597229480743408,
+      "logps/chosen": -205.22775268554688,
+      "logps/pi_response": -108.34332275390625,
       "logps/ref_response": -99.59819793701172,
-      "logps/rejected": -318.347900390625,
-      "loss": 0.6183,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.6433443427085876,
-      "rewards/margins": 0.3338954746723175,
-      "rewards/rejected": -0.9772397875785828,
+      "logps/rejected": -236.5616912841797,
+      "loss": 0.6718,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.09881611168384552,
+      "rewards/margins": 0.06056177616119385,
+      "rewards/rejected": -0.15937790274620056,
       "step": 80
     },
     {
       "epoch": 0.56,
-      "learning_rate": 2.3627616503391812e-07,
-      "logits/chosen": -2.153876781463623,
-      "logits/rejected": -2.1103360652923584,
-      "logps/chosen": -281.10174560546875,
-      "logps/pi_response": -190.38560485839844,
+      "learning_rate": 2.362761650339181e-08,
+      "logits/chosen": -2.681976556777954,
+      "logits/rejected": -2.656203508377075,
+      "logps/chosen": -221.44039916992188,
+      "logps/pi_response": -112.46112060546875,
       "logps/ref_response": -100.86174011230469,
-      "logps/rejected": -323.5328369140625,
-      "loss": 0.6074,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.7317761182785034,
-      "rewards/margins": 0.3521662950515747,
-      "rewards/rejected": -1.0839424133300781,
+      "logps/rejected": -234.1958465576172,
+      "loss": 0.6698,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.13516300916671753,
+      "rewards/margins": 0.055410224944353104,
+      "rewards/rejected": -0.19057324528694153,
       "step": 90
     },
     {
       "epoch": 0.63,
-      "learning_rate": 1.8220596619089573e-07,
-      "logits/chosen": -2.0641961097717285,
-      "logits/rejected": -2.054036855697632,
-      "logps/chosen": -278.60418701171875,
-      "logps/pi_response": -199.0910186767578,
+      "learning_rate": 1.8220596619089574e-08,
+      "logits/chosen": -2.590778112411499,
+      "logits/rejected": -2.5902531147003174,
+      "logps/chosen": -220.1934814453125,
+      "logps/pi_response": -122.29930114746094,
       "logps/ref_response": -112.6884765625,
-      "logps/rejected": -361.7357482910156,
-      "loss": 0.5802,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.702731728553772,
-      "rewards/margins": 0.3770575523376465,
-      "rewards/rejected": -1.0797892808914185,
+      "logps/rejected": -272.1497802734375,
+      "loss": 0.6681,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.11862450838088989,
+      "rewards/margins": 0.06530530005693436,
+      "rewards/rejected": -0.18392980098724365,
       "step": 100
     },
     {
       "epoch": 0.69,
-      "learning_rate": 1.3139467229135998e-07,
-      "logits/chosen": -2.1013143062591553,
-      "logits/rejected": -2.118812084197998,
-      "logps/chosen": -295.019287109375,
-      "logps/pi_response": -188.98532104492188,
+      "learning_rate": 1.3139467229135997e-08,
+      "logits/chosen": -2.6394224166870117,
+      "logits/rejected": -2.660050392150879,
+      "logps/chosen": -230.77310180664062,
+      "logps/pi_response": -112.18680572509766,
       "logps/ref_response": -100.60249328613281,
-      "logps/rejected": -319.04107666015625,
-      "loss": 0.609,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.7831574082374573,
-      "rewards/margins": 0.30071696639060974,
-      "rewards/rejected": -1.0838744640350342,
+      "logps/rejected": -226.88265991210938,
+      "loss": 0.6736,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.14069566130638123,
+      "rewards/margins": 0.02159450575709343,
+      "rewards/rejected": -0.16229018568992615,
       "step": 110
     },
     {
       "epoch": 0.75,
-      "learning_rate": 8.628481651367875e-08,
-      "logits/chosen": -2.1021554470062256,
-      "logits/rejected": -2.029229164123535,
-      "logps/chosen": -324.81427001953125,
-      "logps/pi_response": -228.2621612548828,
+      "learning_rate": 8.628481651367876e-09,
+      "logits/chosen": -2.631422519683838,
+      "logits/rejected": -2.5770680904388428,
+      "logps/chosen": -250.26956176757812,
+      "logps/pi_response": -141.0382537841797,
       "logps/ref_response": -130.39340209960938,
-      "logps/rejected": -352.5882263183594,
-      "loss": 0.5959,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.8705085515975952,
-      "rewards/margins": 0.27299386262893677,
-      "rewards/rejected": -1.1435024738311768,
+      "logps/rejected": -256.5991516113281,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.12506115436553955,
+      "rewards/margins": 0.058550190180540085,
+      "rewards/rejected": -0.18361134827136993,
       "step": 120
     },
     {
       "epoch": 0.82,
-      "learning_rate": 4.904486005914027e-08,
-      "logits/chosen": -2.1313319206237793,
-      "logits/rejected": -2.1133193969726562,
-      "logps/chosen": -308.974853515625,
-      "logps/pi_response": -221.5767822265625,
+      "learning_rate": 4.904486005914027e-09,
+      "logits/chosen": -2.669673442840576,
+      "logits/rejected": -2.6708672046661377,
+      "logps/chosen": -225.036865234375,
+      "logps/pi_response": -123.17768859863281,
       "logps/ref_response": -113.8158950805664,
-      "logps/rejected": -365.41943359375,
-      "loss": 0.5894,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.9527992010116577,
-      "rewards/margins": 0.3640584349632263,
-      "rewards/rejected": -1.3168575763702393,
+      "logps/rejected": -251.93417358398438,
+      "loss": 0.67,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.11341943591833115,
+      "rewards/margins": 0.06858594715595245,
+      "rewards/rejected": -0.182005375623703,
       "step": 130
     },
     {
       "epoch": 0.88,
-      "learning_rate": 2.1464952759020856e-08,
-      "logits/chosen": -2.134899377822876,
-      "logits/rejected": -2.0653326511383057,
-      "logps/chosen": -327.49041748046875,
-      "logps/pi_response": -244.4065704345703,
+      "learning_rate": 2.1464952759020853e-09,
+      "logits/chosen": -2.6793627738952637,
+      "logits/rejected": -2.6371798515319824,
+      "logps/chosen": -242.1341552734375,
+      "logps/pi_response": -131.11148071289062,
       "logps/ref_response": -119.49796295166016,
-      "logps/rejected": -372.51654052734375,
-      "loss": 0.5921,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.9910666346549988,
-      "rewards/margins": 0.4028702676296234,
-      "rewards/rejected": -1.3939368724822998,
+      "logps/rejected": -253.3504180908203,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.13750413060188293,
+      "rewards/margins": 0.06477151811122894,
+      "rewards/rejected": -0.20227563381195068,
       "step": 140
     },
     {
       "epoch": 0.94,
-      "learning_rate": 4.8708793644441086e-09,
-      "logits/chosen": -2.085991382598877,
-      "logits/rejected": -2.056488037109375,
-      "logps/chosen": -301.9818420410156,
-      "logps/pi_response": -204.95913696289062,
+      "learning_rate": 4.870879364444108e-10,
+      "logits/chosen": -2.6344664096832275,
+      "logits/rejected": -2.61822509765625,
+      "logps/chosen": -228.68997192382812,
+      "logps/pi_response": -112.9517593383789,
       "logps/ref_response": -101.72931671142578,
-      "logps/rejected": -332.7224426269531,
-      "loss": 0.5777,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.8675921559333801,
-      "rewards/margins": 0.3387220501899719,
-      "rewards/rejected": -1.2063143253326416,
+      "logps/rejected": -230.86636352539062,
+      "loss": 0.6685,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.13467349112033844,
+      "rewards/margins": 0.053079742938280106,
+      "rewards/rejected": -0.18775323033332825,
       "step": 150
     },
     {
       "epoch": 1.0,
       "step": 159,
       "total_flos": 0.0,
-      "train_loss": 0.6196039487730782,
-      "train_runtime": 4544.1519,
-      "train_samples_per_second": 4.484,
+      "train_loss": 0.6778795974059675,
+      "train_runtime": 4527.8424,
+      "train_samples_per_second": 4.501,
       "train_steps_per_second": 0.035
     }
   ],