Hugging Face's logo

wxzhang
/
dpo-selective-buffer-spo-shift

Text Generation
Transformers
Safetensors
mistral
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
wxzhang commited on
Commit
c6b731c
·
verified ·
1 Parent(s): 82f7292

Model save

Browse files
Files changed (9) hide show
  1. README.md +14 -14
  2. all_results.json +17 -17
  3. eval_results.json +14 -14
  4. model-00001-of-00003.safetensors +1 -1
  5. model-00002-of-00003.safetensors +1 -1
  6. model-00003-of-00003.safetensors +1 -1
  7. train_results.json +3 -3
  8. trainer_state.json +0 -0
  9. training_args.bin +1 -1
README.md CHANGED
@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
- - Loss: 3740.8657
19
- - Rewards/chosen: -0.7639
20
- - Rewards/rejected: -0.8006
21
- - Rewards/accuracies: 0.5440
22
- - Rewards/margins: 0.0367
23
- - Rewards/safe Rewards: -0.7642
24
- - Rewards/unsafe Rewards: -0.7631
25
- - Logps/rejected: -172.4103
26
- - Logps/chosen: -207.2579
27
- - Logits/rejected: -0.8679
28
- - Logits/chosen: -1.2929
29
 
30
  ## Model description
31
 
@@ -62,9 +62,9 @@ The following hyperparameters were used during training:
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
- | 10014.0016 | 0.27 | 500 | 3855.2058 | -0.6234 | -0.6551 | 0.5693 | 0.0317 | -0.6247 | -0.6222 | -157.8614 | -193.2104 | -1.4395 | -1.7663 |
66
- | 9326.0547 | 0.54 | 1000 | 3789.4490 | -0.6776 | -0.7269 | 0.5769 | 0.0494 | -0.6763 | -0.6752 | -165.0468 | -198.6211 | -1.0852 | -1.4592 |
67
- | 8544.8563 | 0.81 | 1500 | 3742.0378 | -0.7610 | -0.7967 | 0.5451 | 0.0358 | -0.7614 | -0.7604 | -172.0258 | -206.9610 | -0.7429 | -1.1929 |
68
 
69
 
70
  ### Framework versions
 
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
+ - Loss: 3023.6533
19
+ - Rewards/chosen: -0.6906
20
+ - Rewards/rejected: -0.6752
21
+ - Rewards/accuracies: 0.4671
22
+ - Rewards/margins: -0.0155
23
+ - Rewards/safe Rewards: -0.6907
24
+ - Rewards/unsafe Rewards: -0.6886
25
+ - Logps/rejected: -159.8684
26
+ - Logps/chosen: -199.9275
27
+ - Logits/rejected: -1.4133
28
+ - Logits/chosen: -1.7139
29
 
30
  ## Model description
31
 
 
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 6785.8539 | 0.27 | 500 | 3119.0669 | -0.6463 | -0.6344 | 0.4765 | -0.0119 | -0.6475 | -0.6439 | -155.7914 | -195.4988 | -1.4656 | -1.7600 |
66
+ | 6458.732 | 0.54 | 1000 | 3057.4275 | -0.6406 | -0.6321 | 0.4848 | -0.0085 | -0.6397 | -0.6374 | -155.5639 | -194.9297 | -1.4110 | -1.7036 |
67
+ | 5925.3711 | 0.81 | 1500 | 3026.4443 | -0.7053 | -0.6899 | 0.4667 | -0.0154 | -0.7055 | -0.7034 | -161.3376 | -201.3946 | -1.3719 | -1.6829 |
68
 
69
 
70
  ### Framework versions
all_results.json CHANGED
@@ -1,23 +1,23 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": -1.2928965091705322,
4
- "eval_logits/rejected": -0.8678836226463318,
5
- "eval_logps/chosen": -207.25787353515625,
6
- "eval_logps/rejected": -172.41029357910156,
7
- "eval_loss": 3740.86572265625,
8
- "eval_rewards/accuracies": 0.5439859628677368,
9
- "eval_rewards/chosen": -0.7639199495315552,
10
- "eval_rewards/margins": 0.036664050072431564,
11
- "eval_rewards/rejected": -0.8005839586257935,
12
- "eval_rewards/safe_rewards": -0.7641976475715637,
13
- "eval_rewards/unsafe_rewards": -0.7631149291992188,
14
- "eval_runtime": 1813.7376,
15
  "eval_samples": 33044,
16
- "eval_samples_per_second": 18.219,
17
- "eval_steps_per_second": 1.139,
18
- "train_loss": 9735.328284782023,
19
- "train_runtime": 39733.2407,
20
  "train_samples": 59478,
21
- "train_samples_per_second": 1.497,
22
  "train_steps_per_second": 0.047
23
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -1.7139310836791992,
4
+ "eval_logits/rejected": -1.4132570028305054,
5
+ "eval_logps/chosen": -199.92747497558594,
6
+ "eval_logps/rejected": -159.868408203125,
7
+ "eval_loss": 3023.6533203125,
8
+ "eval_rewards/accuracies": 0.46708616614341736,
9
+ "eval_rewards/chosen": -0.6906158924102783,
10
+ "eval_rewards/margins": -0.015450715087354183,
11
+ "eval_rewards/rejected": -0.6751651763916016,
12
+ "eval_rewards/safe_rewards": -0.6906617283821106,
13
+ "eval_rewards/unsafe_rewards": -0.6885586977005005,
14
+ "eval_runtime": 1795.035,
15
  "eval_samples": 33044,
16
+ "eval_samples_per_second": 18.409,
17
+ "eval_steps_per_second": 1.151,
18
+ "train_loss": 6725.912355355221,
19
+ "train_runtime": 39534.0439,
20
  "train_samples": 59478,
21
+ "train_samples_per_second": 1.504,
22
  "train_steps_per_second": 0.047
23
  }
eval_results.json CHANGED
@@ -1,18 +1,18 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": -1.2928965091705322,
4
- "eval_logits/rejected": -0.8678836226463318,
5
- "eval_logps/chosen": -207.25787353515625,
6
- "eval_logps/rejected": -172.41029357910156,
7
- "eval_loss": 3740.86572265625,
8
- "eval_rewards/accuracies": 0.5439859628677368,
9
- "eval_rewards/chosen": -0.7639199495315552,
10
- "eval_rewards/margins": 0.036664050072431564,
11
- "eval_rewards/rejected": -0.8005839586257935,
12
- "eval_rewards/safe_rewards": -0.7641976475715637,
13
- "eval_rewards/unsafe_rewards": -0.7631149291992188,
14
- "eval_runtime": 1813.7376,
15
  "eval_samples": 33044,
16
- "eval_samples_per_second": 18.219,
17
- "eval_steps_per_second": 1.139
18
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -1.7139310836791992,
4
+ "eval_logits/rejected": -1.4132570028305054,
5
+ "eval_logps/chosen": -199.92747497558594,
6
+ "eval_logps/rejected": -159.868408203125,
7
+ "eval_loss": 3023.6533203125,
8
+ "eval_rewards/accuracies": 0.46708616614341736,
9
+ "eval_rewards/chosen": -0.6906158924102783,
10
+ "eval_rewards/margins": -0.015450715087354183,
11
+ "eval_rewards/rejected": -0.6751651763916016,
12
+ "eval_rewards/safe_rewards": -0.6906617283821106,
13
+ "eval_rewards/unsafe_rewards": -0.6885586977005005,
14
+ "eval_runtime": 1795.035,
15
  "eval_samples": 33044,
16
+ "eval_samples_per_second": 18.409,
17
+ "eval_steps_per_second": 1.151
18
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:32f392932e66dbbd80cd15c45b886d80041bdfaac756b0f5ce84c3921bbb89a3
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01237913dca4e700daa4070cc683b6c13eda89cf78d0c54d3d09c82bdaa0842a
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:39748a78588d9894770029044032337e80623cf2ad479bc936993e5d61d65011
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ac1f6a38f78078267979eae09aa2b664f60205c937f1fca757e3921367dcc91
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:115bb90a8d5f62db357db0f1d1cebbee0a012ba5b1e181722e4eca35de7bcd4b
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fdfc8fe0bf89b7b810c08d39643a79f057791b632edd8ed43ac59f0f49be6560
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 9735.328284782023,
4
- "train_runtime": 39733.2407,
5
  "train_samples": 59478,
6
- "train_samples_per_second": 1.497,
7
  "train_steps_per_second": 0.047
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 6725.912355355221,
4
+ "train_runtime": 39534.0439,
5
  "train_samples": 59478,
6
+ "train_samples_per_second": 1.504,
7
  "train_steps_per_second": 0.047
8
  }
trainer_state.json CHANGED
The diff for this file is too large to render. See raw diff
 
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a171d71a3e50a657029cd7e2b734057d11bf745dcfd7af4cb39d8fb40be154da
3
  size 6200
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ed17eaa36b3513ff95ee3b96b854d0972cbc8b033863f035e366e1ef021a0ad
3
  size 6200