Hugging Face's logo

wxzhang
/
dpo-selective-buffer-spo-shift

Text Generation
Transformers
Safetensors
mistral
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
wxzhang commited on
Commit
82f7292
·
verified ·
1 Parent(s): 45a0142

Model save

Browse files
Files changed (9) hide show
  1. README.md +14 -14
  2. all_results.json +17 -17
  3. eval_results.json +14 -14
  4. model-00001-of-00003.safetensors +1 -1
  5. model-00002-of-00003.safetensors +1 -1
  6. model-00003-of-00003.safetensors +1 -1
  7. train_results.json +3 -3
  8. trainer_state.json +0 -0
  9. training_args.bin +1 -1
README.md CHANGED
@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
- - Loss: 32.5885
19
- - Rewards/chosen: -0.1837
20
- - Rewards/rejected: -0.1337
21
- - Rewards/accuracies: 0.4256
22
- - Rewards/margins: -0.0500
23
- - Rewards/safe Rewards: -0.1819
24
- - Rewards/unsafe Rewards: -0.1795
25
- - Logps/rejected: -93.6891
26
- - Logps/chosen: -132.7033
27
- - Logits/rejected: -1.7025
28
- - Logits/chosen: -1.9659
29
 
30
  ## Model description
31
 
@@ -62,9 +62,9 @@ The following hyperparameters were used during training:
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
- | 127.076 | 0.27 | 500 | 33.6239 | -0.1345 | -0.0883 | 0.4233 | -0.0461 | -0.1334 | -0.1304 | -93.2350 | -132.2104 | -1.6706 | -1.9417 |
66
- | 97.4026 | 0.54 | 1000 | 33.0573 | -0.1404 | -0.0909 | 0.4196 | -0.0495 | -0.1392 | -0.1366 | -93.2606 | -132.2694 | -1.6791 | -1.9451 |
67
- | 129.5189 | 0.81 | 1500 | 32.6552 | -0.1936 | -0.1481 | 0.4347 | -0.0455 | -0.1916 | -0.1891 | -93.8326 | -132.8015 | -1.7046 | -1.9675 |
68
 
69
 
70
  ### Framework versions
 
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
+ - Loss: 3740.8657
19
+ - Rewards/chosen: -0.7639
20
+ - Rewards/rejected: -0.8006
21
+ - Rewards/accuracies: 0.5440
22
+ - Rewards/margins: 0.0367
23
+ - Rewards/safe Rewards: -0.7642
24
+ - Rewards/unsafe Rewards: -0.7631
25
+ - Logps/rejected: -172.4103
26
+ - Logps/chosen: -207.2579
27
+ - Logits/rejected: -0.8679
28
+ - Logits/chosen: -1.2929
29
 
30
  ## Model description
31
 
 
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 10014.0016 | 0.27 | 500 | 3855.2058 | -0.6234 | -0.6551 | 0.5693 | 0.0317 | -0.6247 | -0.6222 | -157.8614 | -193.2104 | -1.4395 | -1.7663 |
66
+ | 9326.0547 | 0.54 | 1000 | 3789.4490 | -0.6776 | -0.7269 | 0.5769 | 0.0494 | -0.6763 | -0.6752 | -165.0468 | -198.6211 | -1.0852 | -1.4592 |
67
+ | 8544.8563 | 0.81 | 1500 | 3742.0378 | -0.7610 | -0.7967 | 0.5451 | 0.0358 | -0.7614 | -0.7604 | -172.0258 | -206.9610 | -0.7429 | -1.1929 |
68
 
69
 
70
  ### Framework versions
all_results.json CHANGED
@@ -1,23 +1,23 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": -1.9658654928207397,
4
- "eval_logits/rejected": -1.7024726867675781,
5
- "eval_logps/chosen": -132.70326232910156,
6
- "eval_logps/rejected": -93.68914794921875,
7
- "eval_loss": 32.58852767944336,
8
- "eval_rewards/accuracies": 0.42558082938194275,
9
- "eval_rewards/chosen": -0.18373823165893555,
10
- "eval_rewards/margins": -0.05001272261142731,
11
- "eval_rewards/rejected": -0.13372550904750824,
12
- "eval_rewards/safe_rewards": -0.18194682896137238,
13
- "eval_rewards/unsafe_rewards": -0.17947207391262054,
14
- "eval_runtime": 1794.8761,
15
  "eval_samples": 33044,
16
- "eval_samples_per_second": 18.41,
17
- "eval_steps_per_second": 1.151,
18
- "train_loss": 168.60366909983853,
19
- "train_runtime": 39525.3905,
20
  "train_samples": 59478,
21
- "train_samples_per_second": 1.505,
22
  "train_steps_per_second": 0.047
23
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -1.2928965091705322,
4
+ "eval_logits/rejected": -0.8678836226463318,
5
+ "eval_logps/chosen": -207.25787353515625,
6
+ "eval_logps/rejected": -172.41029357910156,
7
+ "eval_loss": 3740.86572265625,
8
+ "eval_rewards/accuracies": 0.5439859628677368,
9
+ "eval_rewards/chosen": -0.7639199495315552,
10
+ "eval_rewards/margins": 0.036664050072431564,
11
+ "eval_rewards/rejected": -0.8005839586257935,
12
+ "eval_rewards/safe_rewards": -0.7641976475715637,
13
+ "eval_rewards/unsafe_rewards": -0.7631149291992188,
14
+ "eval_runtime": 1813.7376,
15
  "eval_samples": 33044,
16
+ "eval_samples_per_second": 18.219,
17
+ "eval_steps_per_second": 1.139,
18
+ "train_loss": 9735.328284782023,
19
+ "train_runtime": 39733.2407,
20
  "train_samples": 59478,
21
+ "train_samples_per_second": 1.497,
22
  "train_steps_per_second": 0.047
23
  }
eval_results.json CHANGED
@@ -1,18 +1,18 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": -1.9658654928207397,
4
- "eval_logits/rejected": -1.7024726867675781,
5
- "eval_logps/chosen": -132.70326232910156,
6
- "eval_logps/rejected": -93.68914794921875,
7
- "eval_loss": 32.58852767944336,
8
- "eval_rewards/accuracies": 0.42558082938194275,
9
- "eval_rewards/chosen": -0.18373823165893555,
10
- "eval_rewards/margins": -0.05001272261142731,
11
- "eval_rewards/rejected": -0.13372550904750824,
12
- "eval_rewards/safe_rewards": -0.18194682896137238,
13
- "eval_rewards/unsafe_rewards": -0.17947207391262054,
14
- "eval_runtime": 1794.8761,
15
  "eval_samples": 33044,
16
- "eval_samples_per_second": 18.41,
17
- "eval_steps_per_second": 1.151
18
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -1.2928965091705322,
4
+ "eval_logits/rejected": -0.8678836226463318,
5
+ "eval_logps/chosen": -207.25787353515625,
6
+ "eval_logps/rejected": -172.41029357910156,
7
+ "eval_loss": 3740.86572265625,
8
+ "eval_rewards/accuracies": 0.5439859628677368,
9
+ "eval_rewards/chosen": -0.7639199495315552,
10
+ "eval_rewards/margins": 0.036664050072431564,
11
+ "eval_rewards/rejected": -0.8005839586257935,
12
+ "eval_rewards/safe_rewards": -0.7641976475715637,
13
+ "eval_rewards/unsafe_rewards": -0.7631149291992188,
14
+ "eval_runtime": 1813.7376,
15
  "eval_samples": 33044,
16
+ "eval_samples_per_second": 18.219,
17
+ "eval_steps_per_second": 1.139
18
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1bc5af5c8ec6e097ac1f27aac5cc1eb5d941c2e27ebcde4fcff41afa0b2dc315
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32f392932e66dbbd80cd15c45b886d80041bdfaac756b0f5ce84c3921bbb89a3
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1df6d32a42d7306181658ef46d540f43027d4b746d4e8f5869c359547d87d2e5
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:39748a78588d9894770029044032337e80623cf2ad479bc936993e5d61d65011
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f157c4464fd21fe86a06797ea18f9f7ced6b01232a221c87c02efb47c492f1e1
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:115bb90a8d5f62db357db0f1d1cebbee0a012ba5b1e181722e4eca35de7bcd4b
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 168.60366909983853,
4
- "train_runtime": 39525.3905,
5
  "train_samples": 59478,
6
- "train_samples_per_second": 1.505,
7
  "train_steps_per_second": 0.047
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 9735.328284782023,
4
+ "train_runtime": 39733.2407,
5
  "train_samples": 59478,
6
+ "train_samples_per_second": 1.497,
7
  "train_steps_per_second": 0.047
8
  }
trainer_state.json CHANGED
The diff for this file is too large to render. See raw diff
 
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1fe5f5e41e8301a65dbf8e77cb571d577ca1a6de221c8c22402abda40db0048d
3
  size 6200
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a171d71a3e50a657029cd7e2b734057d11bf745dcfd7af4cb39d8fb40be154da
3
  size 6200