Hugging Face's logo

Gunslinger3D
/
fine-tuning-Phi2-with-webglm-qa-with-lora_4

PEFT
Safetensors
Generated from Trainer
Model card Files Community
Gunslinger3D commited on
Commit
85245ed
·
verified ·
1 Parent(s): 4affbd4

fine-tuning-Phi2-with-webglm-qa-with-lora_4

Browse files
Files changed (3) hide show
  1. README.md +51 -51
  2. adapter_config.json +6 -4
  3. adapter_model.safetensors +2 -2
README.md CHANGED
@@ -16,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
16
 
17
  This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on an unknown dataset.
18
  It achieves the following results on the evaluation set:
19
- - Loss: 0.2392
20
 
21
  ## Model description
22
 
@@ -51,56 +51,56 @@ The following hyperparameters were used during training:
51
 
52
  | Training Loss | Epoch | Step | Validation Loss |
53
  |:-------------:|:-----:|:----:|:---------------:|
54
- | 8.19 | 0.2 | 10 | 7.9966 |
55
- | 8.0261 | 0.4 | 20 | 7.7896 |
56
- | 7.3527 | 0.6 | 30 | 7.2580 |
57
- | 6.9568 | 0.8 | 40 | 5.9952 |
58
- | 5.2411 | 1.0 | 50 | 3.7880 |
59
- | 2.9772 | 1.2 | 60 | 1.8751 |
60
- | 1.2384 | 1.39 | 70 | 0.7517 |
61
- | 0.6916 | 1.59 | 80 | 0.6684 |
62
- | 0.5669 | 1.79 | 90 | 0.6138 |
63
- | 0.5195 | 1.99 | 100 | 0.5846 |
64
- | 0.5281 | 2.19 | 110 | 0.5607 |
65
- | 0.4764 | 2.39 | 120 | 0.5396 |
66
- | 0.4655 | 2.59 | 130 | 0.5190 |
67
- | 0.4787 | 2.79 | 140 | 0.4980 |
68
- | 0.427 | 2.99 | 150 | 0.4765 |
69
- | 0.41 | 3.19 | 160 | 0.4547 |
70
- | 0.397 | 3.39 | 170 | 0.4317 |
71
- | 0.3648 | 3.59 | 180 | 0.4087 |
72
- | 0.3436 | 3.78 | 190 | 0.3863 |
73
- | 0.3415 | 3.98 | 200 | 0.3661 |
74
- | 0.3072 | 4.18 | 210 | 0.3481 |
75
- | 0.2681 | 4.38 | 220 | 0.3341 |
76
- | 0.3068 | 4.58 | 230 | 0.3201 |
77
- | 0.2526 | 4.78 | 240 | 0.3095 |
78
- | 0.2632 | 4.98 | 250 | 0.3003 |
79
- | 0.2693 | 5.18 | 260 | 0.2936 |
80
- | 0.2194 | 5.38 | 270 | 0.2874 |
81
- | 0.2474 | 5.58 | 280 | 0.2826 |
82
- | 0.2467 | 5.78 | 290 | 0.2770 |
83
- | 0.2188 | 5.98 | 300 | 0.2726 |
84
- | 0.2305 | 6.18 | 310 | 0.2690 |
85
- | 0.2336 | 6.37 | 320 | 0.2643 |
86
- | 0.2192 | 6.57 | 330 | 0.2614 |
87
- | 0.2189 | 6.77 | 340 | 0.2588 |
88
- | 0.2049 | 6.97 | 350 | 0.2564 |
89
- | 0.2096 | 7.17 | 360 | 0.2540 |
90
- | 0.221 | 7.37 | 370 | 0.2521 |
91
- | 0.2167 | 7.57 | 380 | 0.2498 |
92
- | 0.203 | 7.77 | 390 | 0.2484 |
93
- | 0.1999 | 7.97 | 400 | 0.2469 |
94
- | 0.1888 | 8.17 | 410 | 0.2458 |
95
- | 0.195 | 8.37 | 420 | 0.2443 |
96
- | 0.2358 | 8.57 | 430 | 0.2429 |
97
- | 0.1929 | 8.76 | 440 | 0.2419 |
98
- | 0.2066 | 8.96 | 450 | 0.2412 |
99
- | 0.2101 | 9.16 | 460 | 0.2407 |
100
- | 0.2009 | 9.36 | 470 | 0.2400 |
101
- | 0.1976 | 9.56 | 480 | 0.2394 |
102
- | 0.2013 | 9.76 | 490 | 0.2392 |
103
- | 0.1956 | 9.96 | 500 | 0.2392 |
104
 
105
 
106
  ### Framework versions
 
16
 
17
  This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on an unknown dataset.
18
  It achieves the following results on the evaluation set:
19
+ - Loss: 0.1176
20
 
21
  ## Model description
22
 
 
51
 
52
  | Training Loss | Epoch | Step | Validation Loss |
53
  |:-------------:|:-----:|:----:|:---------------:|
54
+ | 8.1178 | 0.2 | 10 | 7.7550 |
55
+ | 7.3762 | 0.4 | 20 | 6.3827 |
56
+ | 4.9217 | 0.6 | 30 | 3.2172 |
57
+ | 1.7792 | 0.8 | 40 | 0.6700 |
58
+ | 0.5779 | 1.0 | 50 | 0.5969 |
59
+ | 0.4824 | 1.2 | 60 | 0.5149 |
60
+ | 0.4689 | 1.39 | 70 | 0.4440 |
61
+ | 0.3833 | 1.59 | 80 | 0.3862 |
62
+ | 0.2916 | 1.79 | 90 | 0.3364 |
63
+ | 0.2435 | 1.99 | 100 | 0.3013 |
64
+ | 0.2538 | 2.19 | 110 | 0.2779 |
65
+ | 0.2147 | 2.39 | 120 | 0.2619 |
66
+ | 0.1982 | 2.59 | 130 | 0.2453 |
67
+ | 0.2183 | 2.79 | 140 | 0.2275 |
68
+ | 0.1737 | 2.99 | 150 | 0.2148 |
69
+ | 0.1794 | 3.19 | 160 | 0.2068 |
70
+ | 0.1692 | 3.39 | 170 | 0.1949 |
71
+ | 0.1573 | 3.59 | 180 | 0.1864 |
72
+ | 0.1478 | 3.78 | 190 | 0.1788 |
73
+ | 0.164 | 3.98 | 200 | 0.1732 |
74
+ | 0.1454 | 4.18 | 210 | 0.1676 |
75
+ | 0.1279 | 4.38 | 220 | 0.1653 |
76
+ | 0.1544 | 4.58 | 230 | 0.1595 |
77
+ | 0.1206 | 4.78 | 240 | 0.1524 |
78
+ | 0.1334 | 4.98 | 250 | 0.1486 |
79
+ | 0.1342 | 5.18 | 260 | 0.1472 |
80
+ | 0.1061 | 5.38 | 270 | 0.1442 |
81
+ | 0.1265 | 5.58 | 280 | 0.1427 |
82
+ | 0.131 | 5.78 | 290 | 0.1389 |
83
+ | 0.1067 | 5.98 | 300 | 0.1374 |
84
+ | 0.1158 | 6.18 | 310 | 0.1331 |
85
+ | 0.1114 | 6.37 | 320 | 0.1323 |
86
+ | 0.1104 | 6.57 | 330 | 0.1311 |
87
+ | 0.108 | 6.77 | 340 | 0.1281 |
88
+ | 0.1015 | 6.97 | 350 | 0.1271 |
89
+ | 0.1 | 7.17 | 360 | 0.1262 |
90
+ | 0.1091 | 7.37 | 370 | 0.1242 |
91
+ | 0.1013 | 7.57 | 380 | 0.1230 |
92
+ | 0.1074 | 7.77 | 390 | 0.1233 |
93
+ | 0.0946 | 7.97 | 400 | 0.1226 |
94
+ | 0.0854 | 8.17 | 410 | 0.1222 |
95
+ | 0.0914 | 8.37 | 420 | 0.1205 |
96
+ | 0.1117 | 8.57 | 430 | 0.1198 |
97
+ | 0.0922 | 8.76 | 440 | 0.1194 |
98
+ | 0.1012 | 8.96 | 450 | 0.1185 |
99
+ | 0.0964 | 9.16 | 460 | 0.1185 |
100
+ | 0.0948 | 9.36 | 470 | 0.1181 |
101
+ | 0.0943 | 9.56 | 480 | 0.1178 |
102
+ | 0.0915 | 9.76 | 490 | 0.1176 |
103
+ | 0.0924 | 9.96 | 500 | 0.1176 |
104
 
105
 
106
  ### Framework versions
adapter_config.json CHANGED
@@ -9,20 +9,22 @@
9
  "layers_pattern": null,
10
  "layers_to_transform": null,
11
  "loftq_config": {},
12
- "lora_alpha": 16,
13
  "lora_dropout": 0.05,
14
  "megatron_config": null,
15
  "megatron_core": "megatron.core",
16
  "modules_to_save": null,
17
  "peft_type": "LORA",
18
- "r": 8,
19
  "rank_pattern": {},
20
  "revision": null,
21
  "target_modules": [
22
  "q_proj",
 
 
 
23
  "v_proj",
24
- "o_proj",
25
- "k_proj"
26
  ],
27
  "task_type": "CAUSAL_LM"
28
  }
 
9
  "layers_pattern": null,
10
  "layers_to_transform": null,
11
  "loftq_config": {},
12
+ "lora_alpha": 32,
13
  "lora_dropout": 0.05,
14
  "megatron_config": null,
15
  "megatron_core": "megatron.core",
16
  "modules_to_save": null,
17
  "peft_type": "LORA",
18
+ "r": 16,
19
  "rank_pattern": {},
20
  "revision": null,
21
  "target_modules": [
22
  "q_proj",
23
+ "fc1",
24
+ "fc2",
25
+ "k_proj",
26
  "v_proj",
27
+ "dense"
 
28
  ],
29
  "task_type": "CAUSAL_LM"
30
  }
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d874c79eebb737b0ee2e68dcfe6c2191dccad0bbcffcbdf1fea45bfd0f89a1eb
3
- size 15754072
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e662e15db52fcb238ea6c11dbfbd4e19c2db88459a035c5c402aa17514af1685
3
+ size 94422368