Hugging Face's logo

wxzhang
/
dpo-selective-buffer-spo-shift

Text Generation
Transformers
Safetensors
mistral
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
wxzhang commited on
Commit
5d84c8e
·
verified ·
1 Parent(s): c6b731c

Model save

Browse files
Files changed (9) hide show
  1. README.md +14 -14
  2. all_results.json +18 -18
  3. eval_results.json +14 -14
  4. model-00001-of-00003.safetensors +1 -1
  5. model-00002-of-00003.safetensors +1 -1
  6. model-00003-of-00003.safetensors +1 -1
  7. train_results.json +4 -4
  8. trainer_state.json +0 -0
  9. training_args.bin +1 -1
README.md CHANGED
@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
- - Loss: 3023.6533
19
- - Rewards/chosen: -0.6906
20
- - Rewards/rejected: -0.6752
21
- - Rewards/accuracies: 0.4671
22
- - Rewards/margins: -0.0155
23
- - Rewards/safe Rewards: -0.6907
24
- - Rewards/unsafe Rewards: -0.6886
25
- - Logps/rejected: -159.8684
26
- - Logps/chosen: -199.9275
27
- - Logits/rejected: -1.4133
28
- - Logits/chosen: -1.7139
29
 
30
  ## Model description
31
 
@@ -62,9 +62,9 @@ The following hyperparameters were used during training:
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
- | 6785.8539 | 0.27 | 500 | 3119.0669 | -0.6463 | -0.6344 | 0.4765 | -0.0119 | -0.6475 | -0.6439 | -155.7914 | -195.4988 | -1.4656 | -1.7600 |
66
- | 6458.732 | 0.54 | 1000 | 3057.4275 | -0.6406 | -0.6321 | 0.4848 | -0.0085 | -0.6397 | -0.6374 | -155.5639 | -194.9297 | -1.4110 | -1.7036 |
67
- | 5925.3711 | 0.81 | 1500 | 3026.4443 | -0.7053 | -0.6899 | 0.4667 | -0.0154 | -0.7055 | -0.7034 | -161.3376 | -201.3946 | -1.3719 | -1.6829 |
68
 
69
 
70
  ### Framework versions
 
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
+ - Loss: 0.6777
19
+ - Rewards/chosen: -0.1371
20
+ - Rewards/rejected: -0.0830
21
+ - Rewards/accuracies: 0.4693
22
+ - Rewards/margins: -0.0541
23
+ - Rewards/safe Rewards: -0.1332
24
+ - Rewards/unsafe Rewards: -0.1263
25
+ - Logps/rejected: -92.4348
26
+ - Logps/chosen: -131.0029
27
+ - Logits/rejected: -1.8308
28
+ - Logits/chosen: -2.0825
29
 
30
  ## Model description
31
 
 
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 131.6857 | 0.27 | 500 | 0.8894 | -0.1023 | -0.0129 | 0.4546 | -0.0893 | -0.1043 | -0.1017 | -92.3648 | -130.9681 | -1.8032 | -2.0565 |
66
+ | 34.7958 | 0.54 | 1000 | 0.7397 | -0.1263 | -0.1290 | 0.5028 | 0.0026 | -0.1237 | -0.1264 | -92.4809 | -130.9922 | -1.7990 | -2.0551 |
67
+ | 15.9924 | 0.81 | 1500 | 0.6823 | -0.1578 | -0.1077 | 0.4713 | -0.0501 | -0.1557 | -0.1535 | -92.4596 | -131.0237 | -1.8335 | -2.0849 |
68
 
69
 
70
  ### Framework versions
all_results.json CHANGED
@@ -1,23 +1,23 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": -1.7139310836791992,
4
- "eval_logits/rejected": -1.4132570028305054,
5
- "eval_logps/chosen": -199.92747497558594,
6
- "eval_logps/rejected": -159.868408203125,
7
- "eval_loss": 3023.6533203125,
8
- "eval_rewards/accuracies": 0.46708616614341736,
9
- "eval_rewards/chosen": -0.6906158924102783,
10
- "eval_rewards/margins": -0.015450715087354183,
11
- "eval_rewards/rejected": -0.6751651763916016,
12
- "eval_rewards/safe_rewards": -0.6906617283821106,
13
- "eval_rewards/unsafe_rewards": -0.6885586977005005,
14
- "eval_runtime": 1795.035,
15
  "eval_samples": 33044,
16
- "eval_samples_per_second": 18.409,
17
- "eval_steps_per_second": 1.151,
18
- "train_loss": 6725.912355355221,
19
- "train_runtime": 39534.0439,
20
  "train_samples": 59478,
21
- "train_samples_per_second": 1.504,
22
- "train_steps_per_second": 0.047
23
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -2.0824697017669678,
4
+ "eval_logits/rejected": -1.830767273902893,
5
+ "eval_logps/chosen": -131.0029296875,
6
+ "eval_logps/rejected": -92.43484497070312,
7
+ "eval_loss": 0.6776853799819946,
8
+ "eval_rewards/accuracies": 0.4693247675895691,
9
+ "eval_rewards/chosen": -0.13705651462078094,
10
+ "eval_rewards/margins": -0.0540921576321125,
11
+ "eval_rewards/rejected": -0.08296435326337814,
12
+ "eval_rewards/safe_rewards": -0.1332445591688156,
13
+ "eval_rewards/unsafe_rewards": -0.12632378935813904,
14
+ "eval_runtime": 2192.4744,
15
  "eval_samples": 33044,
16
+ "eval_samples_per_second": 15.072,
17
+ "eval_steps_per_second": 0.942,
18
+ "train_loss": 67.04043597990268,
19
+ "train_runtime": 46860.0347,
20
  "train_samples": 59478,
21
+ "train_samples_per_second": 1.269,
22
+ "train_steps_per_second": 0.04
23
  }
eval_results.json CHANGED
@@ -1,18 +1,18 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": -1.7139310836791992,
4
- "eval_logits/rejected": -1.4132570028305054,
5
- "eval_logps/chosen": -199.92747497558594,
6
- "eval_logps/rejected": -159.868408203125,
7
- "eval_loss": 3023.6533203125,
8
- "eval_rewards/accuracies": 0.46708616614341736,
9
- "eval_rewards/chosen": -0.6906158924102783,
10
- "eval_rewards/margins": -0.015450715087354183,
11
- "eval_rewards/rejected": -0.6751651763916016,
12
- "eval_rewards/safe_rewards": -0.6906617283821106,
13
- "eval_rewards/unsafe_rewards": -0.6885586977005005,
14
- "eval_runtime": 1795.035,
15
  "eval_samples": 33044,
16
- "eval_samples_per_second": 18.409,
17
- "eval_steps_per_second": 1.151
18
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -2.0824697017669678,
4
+ "eval_logits/rejected": -1.830767273902893,
5
+ "eval_logps/chosen": -131.0029296875,
6
+ "eval_logps/rejected": -92.43484497070312,
7
+ "eval_loss": 0.6776853799819946,
8
+ "eval_rewards/accuracies": 0.4693247675895691,
9
+ "eval_rewards/chosen": -0.13705651462078094,
10
+ "eval_rewards/margins": -0.0540921576321125,
11
+ "eval_rewards/rejected": -0.08296435326337814,
12
+ "eval_rewards/safe_rewards": -0.1332445591688156,
13
+ "eval_rewards/unsafe_rewards": -0.12632378935813904,
14
+ "eval_runtime": 2192.4744,
15
  "eval_samples": 33044,
16
+ "eval_samples_per_second": 15.072,
17
+ "eval_steps_per_second": 0.942
18
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:01237913dca4e700daa4070cc683b6c13eda89cf78d0c54d3d09c82bdaa0842a
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:005b34c91ebaa026860cca1e704abc2402ba69afd83236ee5d27f7e1e15d031c
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5ac1f6a38f78078267979eae09aa2b664f60205c937f1fca757e3921367dcc91
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe8d611e1e8a1c0bd56081caa5c4f58848991500b8d808474c45e165dcff2970
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fdfc8fe0bf89b7b810c08d39643a79f057791b632edd8ed43ac59f0f49be6560
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ed22c07e58842f8b48f59a68fe182733f40403ad2009f48c3335fc1cbfc9ae0
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 6725.912355355221,
4
- "train_runtime": 39534.0439,
5
  "train_samples": 59478,
6
- "train_samples_per_second": 1.504,
7
- "train_steps_per_second": 0.047
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 67.04043597990268,
4
+ "train_runtime": 46860.0347,
5
  "train_samples": 59478,
6
+ "train_samples_per_second": 1.269,
7
+ "train_steps_per_second": 0.04
8
  }
trainer_state.json CHANGED
The diff for this file is too large to render. See raw diff
 
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7ed17eaa36b3513ff95ee3b96b854d0972cbc8b033863f035e366e1ef021a0ad
3
  size 6200
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f72d18ce63262d3807c3741e2f37efb3616938ca3f3f1a048863334c1a483281
3
  size 6200