Hugging Face's logo

wxzhang
/
dpo-selective-buffer-spo-shift

Text Generation
Transformers
Safetensors
mistral
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Model card Files Community
wxzhang commited on
Commit
45a0142
·
verified ·
1 Parent(s): bc9bace

Model save

Browse files
Files changed (9) hide show
  1. README.md +14 -14
  2. all_results.json +18 -18
  3. eval_results.json +14 -14
  4. model-00001-of-00003.safetensors +1 -1
  5. model-00002-of-00003.safetensors +1 -1
  6. model-00003-of-00003.safetensors +1 -1
  7. train_results.json +4 -4
  8. trainer_state.json +0 -0
  9. training_args.bin +1 -1
README.md CHANGED
@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
- - Loss: 4308.2256
19
- - Rewards/chosen: -0.8873
20
- - Rewards/rejected: -0.7996
21
- - Rewards/accuracies: 0.4215
22
- - Rewards/margins: -0.0877
23
- - Rewards/safe Rewards: -0.8895
24
- - Rewards/unsafe Rewards: -0.8833
25
- - Logps/rejected: -172.3120
26
- - Logps/chosen: -219.5941
27
- - Logits/rejected: -1.7103
28
- - Logits/chosen: -1.9417
29
 
30
  ## Model description
31
 
@@ -62,9 +62,9 @@ The following hyperparameters were used during training:
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
- | 6501.718 | 0.27 | 500 | 4520.0283 | -0.9297 | -0.8389 | 0.4060 | -0.0909 | -0.9318 | -0.9235 | -176.2401 | -223.8398 | -1.9246 | -2.1346 |
66
- | 6216.9039 | 0.54 | 1000 | 4353.9048 | -0.8442 | -0.7623 | 0.4268 | -0.0818 | -0.8458 | -0.8396 | -168.5849 | -215.2810 | -1.6500 | -1.8948 |
67
- | 5636.9297 | 0.81 | 1500 | 4313.2891 | -0.8733 | -0.7884 | 0.4233 | -0.0849 | -0.8757 | -0.8693 | -171.1906 | -218.1948 | -1.6817 | -1.9192 |
68
 
69
 
70
  ### Framework versions
 
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
+ - Loss: 32.5885
19
+ - Rewards/chosen: -0.1837
20
+ - Rewards/rejected: -0.1337
21
+ - Rewards/accuracies: 0.4256
22
+ - Rewards/margins: -0.0500
23
+ - Rewards/safe Rewards: -0.1819
24
+ - Rewards/unsafe Rewards: -0.1795
25
+ - Logps/rejected: -93.6891
26
+ - Logps/chosen: -132.7033
27
+ - Logits/rejected: -1.7025
28
+ - Logits/chosen: -1.9659
29
 
30
  ## Model description
31
 
 
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 127.076 | 0.27 | 500 | 33.6239 | -0.1345 | -0.0883 | 0.4233 | -0.0461 | -0.1334 | -0.1304 | -93.2350 | -132.2104 | -1.6706 | -1.9417 |
66
+ | 97.4026 | 0.54 | 1000 | 33.0573 | -0.1404 | -0.0909 | 0.4196 | -0.0495 | -0.1392 | -0.1366 | -93.2606 | -132.2694 | -1.6791 | -1.9451 |
67
+ | 129.5189 | 0.81 | 1500 | 32.6552 | -0.1936 | -0.1481 | 0.4347 | -0.0455 | -0.1916 | -0.1891 | -93.8326 | -132.8015 | -1.7046 | -1.9675 |
68
 
69
 
70
  ### Framework versions
all_results.json CHANGED
@@ -1,23 +1,23 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": -1.941659927368164,
4
- "eval_logits/rejected": -1.7102560997009277,
5
- "eval_logps/chosen": -219.59405517578125,
6
- "eval_logps/rejected": -172.31195068359375,
7
- "eval_loss": 4308.2255859375,
8
- "eval_rewards/accuracies": 0.421466588973999,
9
- "eval_rewards/chosen": -0.8872818350791931,
10
- "eval_rewards/margins": -0.08768116682767868,
11
- "eval_rewards/rejected": -0.7996007204055786,
12
- "eval_rewards/safe_rewards": -0.8894747495651245,
13
- "eval_rewards/unsafe_rewards": -0.8832540512084961,
14
- "eval_runtime": 1873.206,
15
  "eval_samples": 33044,
16
- "eval_samples_per_second": 17.64,
17
- "eval_steps_per_second": 1.103,
18
- "train_loss": 6442.514866035388,
19
- "train_runtime": 46213.9649,
20
  "train_samples": 59478,
21
- "train_samples_per_second": 1.287,
22
- "train_steps_per_second": 0.04
23
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -1.9658654928207397,
4
+ "eval_logits/rejected": -1.7024726867675781,
5
+ "eval_logps/chosen": -132.70326232910156,
6
+ "eval_logps/rejected": -93.68914794921875,
7
+ "eval_loss": 32.58852767944336,
8
+ "eval_rewards/accuracies": 0.42558082938194275,
9
+ "eval_rewards/chosen": -0.18373823165893555,
10
+ "eval_rewards/margins": -0.05001272261142731,
11
+ "eval_rewards/rejected": -0.13372550904750824,
12
+ "eval_rewards/safe_rewards": -0.18194682896137238,
13
+ "eval_rewards/unsafe_rewards": -0.17947207391262054,
14
+ "eval_runtime": 1794.8761,
15
  "eval_samples": 33044,
16
+ "eval_samples_per_second": 18.41,
17
+ "eval_steps_per_second": 1.151,
18
+ "train_loss": 168.60366909983853,
19
+ "train_runtime": 39525.3905,
20
  "train_samples": 59478,
21
+ "train_samples_per_second": 1.505,
22
+ "train_steps_per_second": 0.047
23
  }
eval_results.json CHANGED
@@ -1,18 +1,18 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": -1.941659927368164,
4
- "eval_logits/rejected": -1.7102560997009277,
5
- "eval_logps/chosen": -219.59405517578125,
6
- "eval_logps/rejected": -172.31195068359375,
7
- "eval_loss": 4308.2255859375,
8
- "eval_rewards/accuracies": 0.421466588973999,
9
- "eval_rewards/chosen": -0.8872818350791931,
10
- "eval_rewards/margins": -0.08768116682767868,
11
- "eval_rewards/rejected": -0.7996007204055786,
12
- "eval_rewards/safe_rewards": -0.8894747495651245,
13
- "eval_rewards/unsafe_rewards": -0.8832540512084961,
14
- "eval_runtime": 1873.206,
15
  "eval_samples": 33044,
16
- "eval_samples_per_second": 17.64,
17
- "eval_steps_per_second": 1.103
18
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -1.9658654928207397,
4
+ "eval_logits/rejected": -1.7024726867675781,
5
+ "eval_logps/chosen": -132.70326232910156,
6
+ "eval_logps/rejected": -93.68914794921875,
7
+ "eval_loss": 32.58852767944336,
8
+ "eval_rewards/accuracies": 0.42558082938194275,
9
+ "eval_rewards/chosen": -0.18373823165893555,
10
+ "eval_rewards/margins": -0.05001272261142731,
11
+ "eval_rewards/rejected": -0.13372550904750824,
12
+ "eval_rewards/safe_rewards": -0.18194682896137238,
13
+ "eval_rewards/unsafe_rewards": -0.17947207391262054,
14
+ "eval_runtime": 1794.8761,
15
  "eval_samples": 33044,
16
+ "eval_samples_per_second": 18.41,
17
+ "eval_steps_per_second": 1.151
18
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:942445b93ceb07d53210e131ae61f083af8d298dd45a564f22adfb4fc93282b4
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1bc5af5c8ec6e097ac1f27aac5cc1eb5d941c2e27ebcde4fcff41afa0b2dc315
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:440aee23670b0481dd84cc90b9cabe0db102f9bb96de7710072675bb563eed02
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1df6d32a42d7306181658ef46d540f43027d4b746d4e8f5869c359547d87d2e5
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e7376ba128eb1305c68942fcec845cdf5f2c962d070234bbd0755597f07c4f00
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f157c4464fd21fe86a06797ea18f9f7ced6b01232a221c87c02efb47c492f1e1
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 6442.514866035388,
4
- "train_runtime": 46213.9649,
5
  "train_samples": 59478,
6
- "train_samples_per_second": 1.287,
7
- "train_steps_per_second": 0.04
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 168.60366909983853,
4
+ "train_runtime": 39525.3905,
5
  "train_samples": 59478,
6
+ "train_samples_per_second": 1.505,
7
+ "train_steps_per_second": 0.047
8
  }
trainer_state.json CHANGED
The diff for this file is too large to render. See raw diff
 
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7ed17eaa36b3513ff95ee3b96b854d0972cbc8b033863f035e366e1ef021a0ad
3
  size 6200
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1fe5f5e41e8301a65dbf8e77cb571d577ca1a6de221c8c22402abda40db0048d
3
  size 6200