Hugging Face's logo

klcsp
/
llama3.1-8b-gpt4o_100k_classification-fft

Text Generation
Transformers
TensorBoard
Safetensors
llama
alignment-handbook
trl
sft
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Metrics Community
chansung commited on
Commit
2afc305
•
1 Parent(s): f5d405e

Model save

Browse files
Files changed (5) hide show
  1. README.md +68 -0
  2. all_results.json +9 -0
  3. generation_config.json +10 -0
  4. train_results.json +9 -0
  5. trainer_state.json +386 -0
README.md ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ license: llama3
4
+ base_model: meta-llama/Meta-Llama-3-8B
5
+ tags:
6
+ - trl
7
+ - sft
8
+ - generated_from_trainer
9
+ datasets:
10
+ - generator
11
+ model-index:
12
+ - name: llama3.1-8b-gpt4o_100k_classification-fft
13
+ results: []
14
+ ---
15
+
16
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
17
+ should probably proofread and complete it, then remove this comment. -->
18
+
19
+ # llama3.1-8b-gpt4o_100k_classification-fft
20
+
21
+ This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) on the generator dataset.
22
+ It achieves the following results on the evaluation set:
23
+ - Loss: 2.2658
24
+
25
+ ## Model description
26
+
27
+ More information needed
28
+
29
+ ## Intended uses & limitations
30
+
31
+ More information needed
32
+
33
+ ## Training and evaluation data
34
+
35
+ More information needed
36
+
37
+ ## Training procedure
38
+
39
+ ### Training hyperparameters
40
+
41
+ The following hyperparameters were used during training:
42
+ - learning_rate: 0.0003
43
+ - train_batch_size: 2
44
+ - eval_batch_size: 2
45
+ - seed: 42
46
+ - distributed_type: multi-GPU
47
+ - num_devices: 8
48
+ - gradient_accumulation_steps: 2
49
+ - total_train_batch_size: 32
50
+ - total_eval_batch_size: 16
51
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
52
+ - lr_scheduler_type: cosine
53
+ - lr_scheduler_warmup_ratio: 0.1
54
+ - num_epochs: 1
55
+
56
+ ### Training results
57
+
58
+ | Training Loss | Epoch | Step | Validation Loss |
59
+ |:-------------:|:-----:|:----:|:---------------:|
60
+ | 1.0692 | 1.0 | 237 | 2.2658 |
61
+
62
+
63
+ ### Framework versions
64
+
65
+ - Transformers 4.45.1
66
+ - Pytorch 2.4.1+cu121
67
+ - Datasets 3.0.1
68
+ - Tokenizers 0.20.0
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "total_flos": 12379572142080.0,
4
+ "train_loss": 1.644804019968218,
5
+ "train_runtime": 4500.3652,
6
+ "train_samples": 92634,
7
+ "train_samples_per_second": 1.682,
8
+ "train_steps_per_second": 0.053
9
+ }
generation_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 128256,
3
+ "do_sample": true,
4
+ "eos_token_id": 128257,
5
+ "max_length": 4096,
6
+ "pad_token_id": 128257,
7
+ "temperature": 0.6,
8
+ "top_p": 0.9,
9
+ "transformers_version": "4.45.1"
10
+ }
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "total_flos": 12379572142080.0,
4
+ "train_loss": 1.644804019968218,
5
+ "train_runtime": 4500.3652,
6
+ "train_samples": 92634,
7
+ "train_samples_per_second": 1.682,
8
+ "train_steps_per_second": 0.053
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,386 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "eval_steps": 500,
6
+ "global_step": 237,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.004219409282700422,
13
+ "grad_norm": 41.47472723630441,
14
+ "learning_rate": 1.2499999999999999e-05,
15
+ "loss": 2.7746,
16
+ "step": 1
17
+ },
18
+ {
19
+ "epoch": 0.02109704641350211,
20
+ "grad_norm": 64.04925382439093,
21
+ "learning_rate": 6.25e-05,
22
+ "loss": 7.0206,
23
+ "step": 5
24
+ },
25
+ {
26
+ "epoch": 0.04219409282700422,
27
+ "grad_norm": 31.695867233164076,
28
+ "learning_rate": 0.000125,
29
+ "loss": 2.0729,
30
+ "step": 10
31
+ },
32
+ {
33
+ "epoch": 0.06329113924050633,
34
+ "grad_norm": 20.697061646205608,
35
+ "learning_rate": 0.00018749999999999998,
36
+ "loss": 2.3708,
37
+ "step": 15
38
+ },
39
+ {
40
+ "epoch": 0.08438818565400844,
41
+ "grad_norm": 27.19753629081583,
42
+ "learning_rate": 0.00025,
43
+ "loss": 2.0771,
44
+ "step": 20
45
+ },
46
+ {
47
+ "epoch": 0.10548523206751055,
48
+ "grad_norm": 98.00456381942149,
49
+ "learning_rate": 0.0002999836847426398,
50
+ "loss": 4.1945,
51
+ "step": 25
52
+ },
53
+ {
54
+ "epoch": 0.12658227848101267,
55
+ "grad_norm": 182.51493263483027,
56
+ "learning_rate": 0.0002994130233112417,
57
+ "loss": 3.4014,
58
+ "step": 30
59
+ },
60
+ {
61
+ "epoch": 0.14767932489451477,
62
+ "grad_norm": 11.535478461037668,
63
+ "learning_rate": 0.00029803014471122853,
64
+ "loss": 2.4622,
65
+ "step": 35
66
+ },
67
+ {
68
+ "epoch": 0.16877637130801687,
69
+ "grad_norm": 20.197984273278912,
70
+ "learning_rate": 0.00029584256634447164,
71
+ "loss": 2.6132,
72
+ "step": 40
73
+ },
74
+ {
75
+ "epoch": 0.189873417721519,
76
+ "grad_norm": 8.178205900726194,
77
+ "learning_rate": 0.0002928621800037197,
78
+ "loss": 1.9102,
79
+ "step": 45
80
+ },
81
+ {
82
+ "epoch": 0.2109704641350211,
83
+ "grad_norm": 5.9959330455646125,
84
+ "learning_rate": 0.00028910518722817866,
85
+ "loss": 1.7519,
86
+ "step": 50
87
+ },
88
+ {
89
+ "epoch": 0.2320675105485232,
90
+ "grad_norm": 13.868140556663864,
91
+ "learning_rate": 0.00028459201123108015,
92
+ "loss": 1.7839,
93
+ "step": 55
94
+ },
95
+ {
96
+ "epoch": 0.25316455696202533,
97
+ "grad_norm": 8.017800813910466,
98
+ "learning_rate": 0.00027934718587800417,
99
+ "loss": 1.6271,
100
+ "step": 60
101
+ },
102
+ {
103
+ "epoch": 0.2742616033755274,
104
+ "grad_norm": 5.37043737910495,
105
+ "learning_rate": 0.0002733992223194766,
106
+ "loss": 1.5652,
107
+ "step": 65
108
+ },
109
+ {
110
+ "epoch": 0.29535864978902954,
111
+ "grad_norm": 3.3275774087311523,
112
+ "learning_rate": 0.00026678045400283336,
113
+ "loss": 1.5245,
114
+ "step": 70
115
+ },
116
+ {
117
+ "epoch": 0.31645569620253167,
118
+ "grad_norm": 4.741642465345151,
119
+ "learning_rate": 0.0002595268609058751,
120
+ "loss": 1.5238,
121
+ "step": 75
122
+ },
123
+ {
124
+ "epoch": 0.33755274261603374,
125
+ "grad_norm": 4.030676855228214,
126
+ "learning_rate": 0.00025167787394778763,
127
+ "loss": 1.5316,
128
+ "step": 80
129
+ },
130
+ {
131
+ "epoch": 0.35864978902953587,
132
+ "grad_norm": 2.119168421995465,
133
+ "learning_rate": 0.000243276160640561,
134
+ "loss": 1.4865,
135
+ "step": 85
136
+ },
137
+ {
138
+ "epoch": 0.379746835443038,
139
+ "grad_norm": 9.504914395443073,
140
+ "learning_rate": 0.0002343673931461171,
141
+ "loss": 1.4966,
142
+ "step": 90
143
+ },
144
+ {
145
+ "epoch": 0.4008438818565401,
146
+ "grad_norm": 1.9138438030380622,
147
+ "learning_rate": 0.000225,
148
+ "loss": 1.5385,
149
+ "step": 95
150
+ },
151
+ {
152
+ "epoch": 0.4219409282700422,
153
+ "grad_norm": 2.0250123306044228,
154
+ "learning_rate": 0.0002152249028512724,
155
+ "loss": 1.4467,
156
+ "step": 100
157
+ },
158
+ {
159
+ "epoch": 0.4430379746835443,
160
+ "grad_norm": 5.253723621821196,
161
+ "learning_rate": 0.0002050952396497135,
162
+ "loss": 1.4458,
163
+ "step": 105
164
+ },
165
+ {
166
+ "epoch": 0.4641350210970464,
167
+ "grad_norm": 1.667081359954707,
168
+ "learning_rate": 0.00019466607578508832,
169
+ "loss": 1.4039,
170
+ "step": 110
171
+ },
172
+ {
173
+ "epoch": 0.48523206751054854,
174
+ "grad_norm": 1.612910217878065,
175
+ "learning_rate": 0.00018399410474875,
176
+ "loss": 1.3915,
177
+ "step": 115
178
+ },
179
+ {
180
+ "epoch": 0.5063291139240507,
181
+ "grad_norm": 4.664851099721208,
182
+ "learning_rate": 0.00017313733994479531,
183
+ "loss": 1.3946,
184
+ "step": 120
185
+ },
186
+ {
187
+ "epoch": 0.5274261603375527,
188
+ "grad_norm": 1.1333666368766908,
189
+ "learning_rate": 0.00016215479932610397,
190
+ "loss": 1.3759,
191
+ "step": 125
192
+ },
193
+ {
194
+ "epoch": 0.5485232067510548,
195
+ "grad_norm": 46.39351482577546,
196
+ "learning_rate": 0.00015110618456959686,
197
+ "loss": 1.3596,
198
+ "step": 130
199
+ },
200
+ {
201
+ "epoch": 0.569620253164557,
202
+ "grad_norm": 11.304078659755062,
203
+ "learning_rate": 0.00014005155653473443,
204
+ "loss": 1.3708,
205
+ "step": 135
206
+ },
207
+ {
208
+ "epoch": 0.5907172995780591,
209
+ "grad_norm": 1.4112479369616122,
210
+ "learning_rate": 0.00012905100876947907,
211
+ "loss": 1.3954,
212
+ "step": 140
213
+ },
214
+ {
215
+ "epoch": 0.6118143459915611,
216
+ "grad_norm": 4.386921961578911,
217
+ "learning_rate": 0.00011816434083856155,
218
+ "loss": 1.3064,
219
+ "step": 145
220
+ },
221
+ {
222
+ "epoch": 0.6329113924050633,
223
+ "grad_norm": 1.2097887430832492,
224
+ "learning_rate": 0.00010745073324985548,
225
+ "loss": 1.3082,
226
+ "step": 150
227
+ },
228
+ {
229
+ "epoch": 0.6540084388185654,
230
+ "grad_norm": 1.1759289220096156,
231
+ "learning_rate": 9.696842574597847e-05,
232
+ "loss": 1.2311,
233
+ "step": 155
234
+ },
235
+ {
236
+ "epoch": 0.6751054852320675,
237
+ "grad_norm": 2.762333419837543,
238
+ "learning_rate": 8.677440070994279e-05,
239
+ "loss": 1.2588,
240
+ "step": 160
241
+ },
242
+ {
243
+ "epoch": 0.6962025316455697,
244
+ "grad_norm": 0.8634167427320883,
245
+ "learning_rate": 7.692407340588055e-05,
246
+ "loss": 1.1888,
247
+ "step": 165
248
+ },
249
+ {
250
+ "epoch": 0.7172995780590717,
251
+ "grad_norm": 2.2322735412619386,
252
+ "learning_rate": 6.747099073871015e-05,
253
+ "loss": 1.2047,
254
+ "step": 170
255
+ },
256
+ {
257
+ "epoch": 0.7383966244725738,
258
+ "grad_norm": 0.7808932928409826,
259
+ "learning_rate": 5.8466540170303925e-05,
260
+ "loss": 1.1594,
261
+ "step": 175
262
+ },
263
+ {
264
+ "epoch": 0.759493670886076,
265
+ "grad_norm": 0.7823146836939375,
266
+ "learning_rate": 4.995967037450238e-05,
267
+ "loss": 1.1763,
268
+ "step": 180
269
+ },
270
+ {
271
+ "epoch": 0.7805907172995781,
272
+ "grad_norm": 0.7925457205036505,
273
+ "learning_rate": 4.199662514951142e-05,
274
+ "loss": 1.1291,
275
+ "step": 185
276
+ },
277
+ {
278
+ "epoch": 0.8016877637130801,
279
+ "grad_norm": 0.8477428122185438,
280
+ "learning_rate": 3.4620692034146904e-05,
281
+ "loss": 1.1162,
282
+ "step": 190
283
+ },
284
+ {
285
+ "epoch": 0.8227848101265823,
286
+ "grad_norm": 0.7180062181554933,
287
+ "learning_rate": 2.7871966994463884e-05,
288
+ "loss": 1.1061,
289
+ "step": 195
290
+ },
291
+ {
292
+ "epoch": 0.8438818565400844,
293
+ "grad_norm": 0.6932161729089131,
294
+ "learning_rate": 2.1787136459944886e-05,
295
+ "loss": 1.1264,
296
+ "step": 200
297
+ },
298
+ {
299
+ "epoch": 0.8649789029535865,
300
+ "grad_norm": 0.7413528530014878,
301
+ "learning_rate": 1.639927789411174e-05,
302
+ "loss": 1.1001,
303
+ "step": 205
304
+ },
305
+ {
306
+ "epoch": 0.8860759493670886,
307
+ "grad_norm": 0.7062510216168646,
308
+ "learning_rate": 1.1737679983668258e-05,
309
+ "loss": 1.0853,
310
+ "step": 210
311
+ },
312
+ {
313
+ "epoch": 0.9071729957805907,
314
+ "grad_norm": 0.6294764868696124,
315
+ "learning_rate": 7.8276834236364e-06,
316
+ "loss": 1.0527,
317
+ "step": 215
318
+ },
319
+ {
320
+ "epoch": 0.9282700421940928,
321
+ "grad_norm": 0.6759047685020939,
322
+ "learning_rate": 4.690543163985771e-06,
323
+ "loss": 1.0758,
324
+ "step": 220
325
+ },
326
+ {
327
+ "epoch": 0.9493670886075949,
328
+ "grad_norm": 1.3620865902651844,
329
+ "learning_rate": 2.343312866591163e-06,
330
+ "loss": 1.0805,
331
+ "step": 225
332
+ },
333
+ {
334
+ "epoch": 0.9704641350210971,
335
+ "grad_norm": 0.6839144968477266,
336
+ "learning_rate": 7.987522006165426e-07,
337
+ "loss": 1.0588,
338
+ "step": 230
339
+ },
340
+ {
341
+ "epoch": 0.9915611814345991,
342
+ "grad_norm": 1.1867918696725697,
343
+ "learning_rate": 6.525748027244593e-08,
344
+ "loss": 1.0692,
345
+ "step": 235
346
+ },
347
+ {
348
+ "epoch": 1.0,
349
+ "eval_loss": 2.265784978866577,
350
+ "eval_runtime": 2.2863,
351
+ "eval_samples_per_second": 0.875,
352
+ "eval_steps_per_second": 0.437,
353
+ "step": 237
354
+ },
355
+ {
356
+ "epoch": 1.0,
357
+ "step": 237,
358
+ "total_flos": 12379572142080.0,
359
+ "train_loss": 1.644804019968218,
360
+ "train_runtime": 4500.3652,
361
+ "train_samples_per_second": 1.682,
362
+ "train_steps_per_second": 0.053
363
+ }
364
+ ],
365
+ "logging_steps": 5,
366
+ "max_steps": 237,
367
+ "num_input_tokens_seen": 0,
368
+ "num_train_epochs": 1,
369
+ "save_steps": 100,
370
+ "stateful_callbacks": {
371
+ "TrainerControl": {
372
+ "args": {
373
+ "should_epoch_stop": false,
374
+ "should_evaluate": false,
375
+ "should_log": false,
376
+ "should_save": true,
377
+ "should_training_stop": true
378
+ },
379
+ "attributes": {}
380
+ }
381
+ },
382
+ "total_flos": 12379572142080.0,
383
+ "train_batch_size": 2,
384
+ "trial_name": null,
385
+ "trial_params": null
386
+ }