Hugging Face's logo

GeorgiaTech
/
0.0_llama_nodpo_3iters_bs128_531lr_iter_2

Text Generation
Transformers
Safetensors
llama
alignment-handbook
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
1
0.0_llama_nodpo_3iters_bs128_531lr_iter_2
File size: 10,630 Bytes 
6d9216a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
 
{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984301412872841,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.006279434850863423,
      "grad_norm": 16.698454749053152,
      "learning_rate": 1.875e-08,
      "logits/chosen": 0.13163629174232483,
      "logits/rejected": 0.7037353515625,
      "logps/chosen": -296.6709289550781,
      "logps/pi_response": -123.40753173828125,
      "logps/ref_response": -123.40753173828125,
      "logps/rejected": -325.5771484375,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06279434850863422,
      "grad_norm": 24.513917430946538,
      "learning_rate": 1.875e-07,
      "logits/chosen": 0.6406950354576111,
      "logits/rejected": 0.8759365081787109,
      "logps/chosen": -260.0070495605469,
      "logps/pi_response": -114.28534698486328,
      "logps/ref_response": -114.47286224365234,
      "logps/rejected": -385.5276184082031,
      "loss": 0.6924,
      "rewards/accuracies": 0.4097222089767456,
      "rewards/chosen": 0.00021976388234179467,
      "rewards/margins": -2.7502783268573694e-05,
      "rewards/rejected": 0.000247266492806375,
      "step": 10
    },
    {
      "epoch": 0.12558869701726844,
      "grad_norm": 20.98441253721937,
      "learning_rate": 2.9942119880575817e-07,
      "logits/chosen": 0.5097376704216003,
      "logits/rejected": 0.8540347814559937,
      "logps/chosen": -268.75433349609375,
      "logps/pi_response": -118.2417221069336,
      "logps/ref_response": -118.39286041259766,
      "logps/rejected": -400.633544921875,
      "loss": 0.6749,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03052676096558571,
      "rewards/margins": 0.040752165019512177,
      "rewards/rejected": -0.07127892971038818,
      "step": 20
    },
    {
      "epoch": 0.18838304552590268,
      "grad_norm": 13.999591521451507,
      "learning_rate": 2.929608750821129e-07,
      "logits/chosen": 0.5199416875839233,
      "logits/rejected": 0.992133617401123,
      "logps/chosen": -295.43292236328125,
      "logps/pi_response": -119.1610336303711,
      "logps/ref_response": -118.39522552490234,
      "logps/rejected": -421.61041259765625,
      "loss": 0.6174,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.17318478226661682,
      "rewards/margins": 0.18207214772701263,
      "rewards/rejected": -0.35525694489479065,
      "step": 30
    },
    {
      "epoch": 0.25117739403453687,
      "grad_norm": 14.705995187750815,
      "learning_rate": 2.7962832564252725e-07,
      "logits/chosen": 0.5350409746170044,
      "logits/rejected": 0.9762212634086609,
      "logps/chosen": -292.2400207519531,
      "logps/pi_response": -122.87149810791016,
      "logps/ref_response": -120.0985336303711,
      "logps/rejected": -484.0975646972656,
      "loss": 0.5739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30772843956947327,
      "rewards/margins": 0.45096302032470703,
      "rewards/rejected": -0.7586914300918579,
      "step": 40
    },
    {
      "epoch": 0.3139717425431711,
      "grad_norm": 20.925472606748368,
      "learning_rate": 2.6006445513357056e-07,
      "logits/chosen": 0.6897233724594116,
      "logits/rejected": 1.0123343467712402,
      "logps/chosen": -320.56976318359375,
      "logps/pi_response": -124.61143493652344,
      "logps/ref_response": -115.71650695800781,
      "logps/rejected": -523.7175903320312,
      "loss": 0.5629,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.6995627284049988,
      "rewards/margins": 0.6693333387374878,
      "rewards/rejected": -1.3688960075378418,
      "step": 50
    },
    {
      "epoch": 0.37676609105180536,
      "grad_norm": 13.008934683020064,
      "learning_rate": 2.3520971200967334e-07,
      "logits/chosen": 0.6137208938598633,
      "logits/rejected": 1.0412781238555908,
      "logps/chosen": -332.25738525390625,
      "logps/pi_response": -126.09577941894531,
      "logps/ref_response": -118.1528549194336,
      "logps/rejected": -495.4088439941406,
      "loss": 0.555,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5905637145042419,
      "rewards/margins": 0.6077089309692383,
      "rewards/rejected": -1.198272705078125,
      "step": 60
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 12.777487677582881,
      "learning_rate": 2.0625888054143427e-07,
      "logits/chosen": 0.593045175075531,
      "logits/rejected": 0.9839151501655579,
      "logps/chosen": -273.0267333984375,
      "logps/pi_response": -126.1861801147461,
      "logps/ref_response": -120.62638854980469,
      "logps/rejected": -531.7461547851562,
      "loss": 0.5431,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3775605261325836,
      "rewards/margins": 0.8122557401657104,
      "rewards/rejected": -1.1898162364959717,
      "step": 70
    },
    {
      "epoch": 0.5023547880690737,
      "grad_norm": 10.709251992827037,
      "learning_rate": 1.7460364672965327e-07,
      "logits/chosen": 0.6686810255050659,
      "logits/rejected": 1.0736128091812134,
      "logps/chosen": -280.6498107910156,
      "logps/pi_response": -112.1661376953125,
      "logps/ref_response": -106.67897033691406,
      "logps/rejected": -489.26556396484375,
      "loss": 0.5321,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4469337463378906,
      "rewards/margins": 0.6373990774154663,
      "rewards/rejected": -1.084332823753357,
      "step": 80
    },
    {
      "epoch": 0.565149136577708,
      "grad_norm": 11.68215452300686,
      "learning_rate": 1.4176569902035086e-07,
      "logits/chosen": 0.6378764510154724,
      "logits/rejected": 1.0353127717971802,
      "logps/chosen": -339.03973388671875,
      "logps/pi_response": -119.71498107910156,
      "logps/ref_response": -111.9307861328125,
      "logps/rejected": -513.7333984375,
      "loss": 0.5119,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.5252664685249329,
      "rewards/margins": 0.7039340734481812,
      "rewards/rejected": -1.2292006015777588,
      "step": 90
    },
    {
      "epoch": 0.6279434850863422,
      "grad_norm": 11.96502204484806,
      "learning_rate": 1.0932357971453743e-07,
      "logits/chosen": 0.7395003437995911,
      "logits/rejected": 1.0328724384307861,
      "logps/chosen": -301.8004455566406,
      "logps/pi_response": -116.29144287109375,
      "logps/ref_response": -108.0909423828125,
      "logps/rejected": -524.360107421875,
      "loss": 0.547,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.5083015561103821,
      "rewards/margins": 0.7502217888832092,
      "rewards/rejected": -1.2585232257843018,
      "step": 100
    },
    {
      "epoch": 0.6907378335949764,
      "grad_norm": 11.40631298798362,
      "learning_rate": 7.883680337481599e-08,
      "logits/chosen": 0.7460795640945435,
      "logits/rejected": 0.9838323593139648,
      "logps/chosen": -305.3519592285156,
      "logps/pi_response": -125.8452377319336,
      "logps/ref_response": -117.07008361816406,
      "logps/rejected": -504.4964294433594,
      "loss": 0.5349,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4922094941139221,
      "rewards/margins": 0.6827653646469116,
      "rewards/rejected": -1.174974799156189,
      "step": 110
    },
    {
      "epoch": 0.7535321821036107,
      "grad_norm": 10.671563097729658,
      "learning_rate": 5.177088990820725e-08,
      "logits/chosen": 0.5097354650497437,
      "logits/rejected": 0.8302543759346008,
      "logps/chosen": -327.8287353515625,
      "logps/pi_response": -134.05953979492188,
      "logps/ref_response": -125.61170959472656,
      "logps/rejected": -554.1288452148438,
      "loss": 0.524,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.5032998323440552,
      "rewards/margins": 0.7790510058403015,
      "rewards/rejected": -1.282350778579712,
      "step": 120
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 13.393410138993277,
      "learning_rate": 2.942691603548416e-08,
      "logits/chosen": 0.5484687089920044,
      "logits/rejected": 1.0104806423187256,
      "logps/chosen": -329.3583984375,
      "logps/pi_response": -133.8960723876953,
      "logps/ref_response": -126.83935546875,
      "logps/rejected": -530.1004028320312,
      "loss": 0.5048,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4904448986053467,
      "rewards/margins": 0.7761750221252441,
      "rewards/rejected": -1.2666199207305908,
      "step": 130
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 12.267577779535525,
      "learning_rate": 1.2878971655412513e-08,
      "logits/chosen": 0.5720739364624023,
      "logits/rejected": 0.9325042963027954,
      "logps/chosen": -310.2190856933594,
      "logps/pi_response": -136.25198364257812,
      "logps/ref_response": -126.86582946777344,
      "logps/rejected": -564.576171875,
      "loss": 0.5164,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.481649786233902,
      "rewards/margins": 0.8537376523017883,
      "rewards/rejected": -1.3353874683380127,
      "step": 140
    },
    {
      "epoch": 0.9419152276295133,
      "grad_norm": 11.331362077198552,
      "learning_rate": 2.922527618666465e-09,
      "logits/chosen": 0.5811373591423035,
      "logits/rejected": 0.9567831158638,
      "logps/chosen": -304.1393737792969,
      "logps/pi_response": -123.76485443115234,
      "logps/ref_response": -114.90129089355469,
      "logps/rejected": -522.5949096679688,
      "loss": 0.5227,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.5134440064430237,
      "rewards/margins": 0.7166833281517029,
      "rewards/rejected": -1.2301273345947266,
      "step": 150
    },
    {
      "epoch": 0.9984301412872841,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 0.5592624436384477,
      "train_runtime": 4324.4895,
      "train_samples_per_second": 4.712,
      "train_steps_per_second": 0.037
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}