Hugging Face's logo

nebchi
/
orion-14B-fusellm-v02

Text Generation
Transformers
PyTorch
Korean
orion
conversational
custom_code
Model card Files Community
nebchi commited on
Commit
bf9da58
·
verified ·
1 Parent(s): 5c04c02

Upload folder using huggingface_hub

Browse files
Files changed (10) hide show
  1. .ipynb_checkpoints/analysis-checkpoint.json +660 -0
  2. analysis.json +660 -0
  3. config.json +33 -0
  4. generation_config.json +13 -0
  5. pytorch_model-00001-of-00002.bin +3 -0
  6. pytorch_model-00002-of-00002.bin +3 -0
  7. pytorch_model.bin.index.json +451 -0
  8. special_tokens_map.json +30 -0
  9. tokenizer.model +3 -0
  10. tokenizer_config.json +45 -0
.ipynb_checkpoints/analysis-checkpoint.json ADDED
@@ -0,0 +1,660 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "aveage_var_degree": 3.892243738108603e-09,
3
+ "total_diff_num": 11385304937,
4
+ "total_param_num": 14498703360,
5
+ "diff_rate": 78.5263664915757,
6
+ "layer_to_params": {
7
+ "0": 1.988666535246705e-08,
8
+ "1": 2.6983774006922925e-08,
9
+ "2": 1.319228530894409e-08,
10
+ "3": 6.557201376281705e-08,
11
+ "4": 4.205192939414228e-08,
12
+ "5": 1.3925403408610472e-08,
13
+ "6": 1.2824742500792588e-08,
14
+ "7": 1.1327664732443272e-08,
15
+ "8": 1.3955765562481565e-08,
16
+ "9": 2.148611166390114e-08,
17
+ "10": 1.7289768219057282e-08,
18
+ "11": 1.329685737538368e-08,
19
+ "12": 1.8387197199242386e-08,
20
+ "13": 1.573349900733471e-08,
21
+ "14": 1.8524112318538675e-08,
22
+ "15": 1.8548633124556574e-08,
23
+ "16": 1.590110241443387e-08,
24
+ "17": 1.5135453732038522e-08,
25
+ "18": 1.1342735059537811e-08,
26
+ "19": 1.0097682785134383e-08,
27
+ "20": 1.248061909883821e-08,
28
+ "21": 1.415265751110913e-08,
29
+ "22": 1.490716693811472e-08,
30
+ "23": 1.6227143244326544e-08,
31
+ "24": 1.0399099193610557e-08,
32
+ "25": 1.4272791352000781e-08,
33
+ "26": 1.2074979475566277e-08,
34
+ "27": 1.75786421745232e-08,
35
+ "28": 1.7063607155750844e-08,
36
+ "29": 1.5309838493256697e-08,
37
+ "30": 2.0628235593752226e-08,
38
+ "31": 2.4866034351936353e-08,
39
+ "32": 2.5866199138135922e-08,
40
+ "33": 3.406494921406588e-08,
41
+ "34": 3.5020059196237226e-08,
42
+ "35": 4.820790008388855e-08,
43
+ "36": 4.640676214287311e-08,
44
+ "37": 3.5461280324019145e-08,
45
+ "38": 3.979688822634368e-08,
46
+ "39": 6.100158140990667e-08
47
+ },
48
+ "key_to_params": {
49
+ "model.embed_tokens.weight": 2.917434234714228e-19,
50
+ "model.layers.0.self_attn.q_proj.weight": 1.187384697232119e-09,
51
+ "model.layers.0.self_attn.k_proj.weight": 1.1349608541877388e-09,
52
+ "model.layers.0.self_attn.v_proj.weight": 3.0414742013817886e-09,
53
+ "model.layers.0.self_attn.o_proj.weight": 2.556667766384635e-09,
54
+ "model.layers.0.mlp.gate_proj.weight": 8.003662514965982e-09,
55
+ "model.layers.0.mlp.up_proj.weight": 2.3990156705622212e-09,
56
+ "model.layers.0.mlp.down_proj.weight": 1.5634996467876287e-09,
57
+ "model.layers.0.input_layernorm.weight": 0.0,
58
+ "model.layers.0.input_layernorm.bias": 2.710505431213761e-19,
59
+ "model.layers.0.post_attention_layernorm.weight": 0.0,
60
+ "model.layers.0.post_attention_layernorm.bias": 6.938893903907229e-19,
61
+ "model.layers.1.self_attn.q_proj.weight": 2.0518091048415955e-09,
62
+ "model.layers.1.self_attn.k_proj.weight": 1.6146566395036643e-09,
63
+ "model.layers.1.self_attn.v_proj.weight": 1.4650281627837103e-08,
64
+ "model.layers.1.self_attn.o_proj.weight": 4.025015698516654e-09,
65
+ "model.layers.1.mlp.gate_proj.weight": 1.7941124497156125e-09,
66
+ "model.layers.1.mlp.up_proj.weight": 2.191004947841672e-09,
67
+ "model.layers.1.mlp.down_proj.weight": 6.568935380831438e-10,
68
+ "model.layers.1.input_layernorm.weight": 0.0,
69
+ "model.layers.1.input_layernorm.bias": 3.9916427639358903e-19,
70
+ "model.layers.1.post_attention_layernorm.weight": 0.0,
71
+ "model.layers.1.post_attention_layernorm.bias": 1.8431436932253575e-19,
72
+ "model.layers.2.self_attn.q_proj.weight": 8.317828559256668e-10,
73
+ "model.layers.2.self_attn.k_proj.weight": 1.0357459956367166e-09,
74
+ "model.layers.2.self_attn.v_proj.weight": 4.803724777957541e-09,
75
+ "model.layers.2.self_attn.o_proj.weight": 2.938668899332697e-09,
76
+ "model.layers.2.mlp.gate_proj.weight": 1.4420785040177482e-09,
77
+ "model.layers.2.mlp.up_proj.weight": 1.6792421320133145e-09,
78
+ "model.layers.2.mlp.down_proj.weight": 4.6104214372159426e-10,
79
+ "model.layers.2.input_layernorm.weight": 0.0,
80
+ "model.layers.2.input_layernorm.bias": 2.9544509200229997e-19,
81
+ "model.layers.2.post_attention_layernorm.weight": 0.0,
82
+ "model.layers.2.post_attention_layernorm.bias": 4.336808689942018e-20,
83
+ "model.layers.3.self_attn.q_proj.weight": 8.837429277264163e-10,
84
+ "model.layers.3.self_attn.k_proj.weight": 1.0162795405221914e-09,
85
+ "model.layers.3.self_attn.v_proj.weight": 3.0072123990976253e-09,
86
+ "model.layers.3.self_attn.o_proj.weight": 4.018537538286182e-09,
87
+ "model.layers.3.mlp.gate_proj.weight": 1.4122169507876e-09,
88
+ "model.layers.3.mlp.up_proj.weight": 1.5453091843179814e-09,
89
+ "model.layers.3.mlp.down_proj.weight": 5.368871522174838e-08,
90
+ "model.layers.3.input_layernorm.weight": 0.0,
91
+ "model.layers.3.input_layernorm.bias": 1.328147661294743e-19,
92
+ "model.layers.3.post_attention_layernorm.weight": 0.0,
93
+ "model.layers.3.post_attention_layernorm.bias": 1.9786689647860455e-19,
94
+ "model.layers.4.self_attn.q_proj.weight": 8.681465857307558e-10,
95
+ "model.layers.4.self_attn.k_proj.weight": 1.7090624737647886e-09,
96
+ "model.layers.4.self_attn.v_proj.weight": 2.7331961405252512e-09,
97
+ "model.layers.4.self_attn.o_proj.weight": 1.8018978664713358e-09,
98
+ "model.layers.4.mlp.gate_proj.weight": 1.7898181188987412e-09,
99
+ "model.layers.4.mlp.up_proj.weight": 1.897217695538226e-09,
100
+ "model.layers.4.mlp.down_proj.weight": 3.12525905125464e-08,
101
+ "model.layers.4.input_layernorm.weight": 0.0,
102
+ "model.layers.4.input_layernorm.bias": 4.797594613248357e-19,
103
+ "model.layers.4.post_attention_layernorm.weight": 0.0,
104
+ "model.layers.4.post_attention_layernorm.bias": 1.8702487475374952e-19,
105
+ "model.layers.5.self_attn.q_proj.weight": 9.632125852476747e-10,
106
+ "model.layers.5.self_attn.k_proj.weight": 7.343454910824221e-10,
107
+ "model.layers.5.self_attn.v_proj.weight": 3.0922541327527143e-09,
108
+ "model.layers.5.self_attn.o_proj.weight": 1.6237197542068315e-09,
109
+ "model.layers.5.mlp.gate_proj.weight": 1.7408918514168667e-09,
110
+ "model.layers.5.mlp.up_proj.weight": 1.8568865319442314e-09,
111
+ "model.layers.5.mlp.down_proj.weight": 3.914093061515208e-09,
112
+ "model.layers.5.input_layernorm.weight": 0.0,
113
+ "model.layers.5.input_layernorm.bias": 0.0,
114
+ "model.layers.5.post_attention_layernorm.weight": 0.0,
115
+ "model.layers.5.post_attention_layernorm.bias": 4.445228907190568e-19,
116
+ "model.layers.6.self_attn.q_proj.weight": 4.4907185525744355e-10,
117
+ "model.layers.6.self_attn.k_proj.weight": 6.927900386699548e-10,
118
+ "model.layers.6.self_attn.v_proj.weight": 2.1942298644717086e-09,
119
+ "model.layers.6.self_attn.o_proj.weight": 2.0572406356222928e-09,
120
+ "model.layers.6.mlp.gate_proj.weight": 3.026063382094435e-09,
121
+ "model.layers.6.mlp.up_proj.weight": 3.793664973272826e-09,
122
+ "model.layers.6.mlp.down_proj.weight": 6.116817511762444e-10,
123
+ "model.layers.6.input_layernorm.weight": 0.0,
124
+ "model.layers.6.input_layernorm.bias": 0.0,
125
+ "model.layers.6.post_attention_layernorm.weight": 0.0,
126
+ "model.layers.6.post_attention_layernorm.bias": 2.2768245622195594e-19,
127
+ "model.layers.7.self_attn.q_proj.weight": 5.192902108319686e-10,
128
+ "model.layers.7.self_attn.k_proj.weight": 4.908492456934255e-10,
129
+ "model.layers.7.self_attn.v_proj.weight": 3.6511215739665203e-09,
130
+ "model.layers.7.self_attn.o_proj.weight": 1.6526492174762098e-09,
131
+ "model.layers.7.mlp.gate_proj.weight": 1.8395443627620504e-09,
132
+ "model.layers.7.mlp.up_proj.weight": 2.646799543981615e-09,
133
+ "model.layers.7.mlp.down_proj.weight": 5.274105774333293e-10,
134
+ "model.layers.7.input_layernorm.weight": 0.0,
135
+ "model.layers.7.input_layernorm.bias": 1.7618285302889447e-19,
136
+ "model.layers.7.post_attention_layernorm.weight": 0.0,
137
+ "model.layers.7.post_attention_layernorm.bias": 1.2197274440461925e-19,
138
+ "model.layers.8.self_attn.q_proj.weight": 6.803252006193361e-10,
139
+ "model.layers.8.self_attn.k_proj.weight": 1.0779841375097021e-09,
140
+ "model.layers.8.self_attn.v_proj.weight": 2.402063898898632e-09,
141
+ "model.layers.8.self_attn.o_proj.weight": 2.236740499483858e-09,
142
+ "model.layers.8.mlp.gate_proj.weight": 4.132192543693236e-09,
143
+ "model.layers.8.mlp.up_proj.weight": 2.8210202647945454e-09,
144
+ "model.layers.8.mlp.down_proj.weight": 6.054390174388875e-10,
145
+ "model.layers.8.input_layernorm.weight": 0.0,
146
+ "model.layers.8.input_layernorm.bias": 4.336808689942018e-20,
147
+ "model.layers.8.post_attention_layernorm.weight": 0.0,
148
+ "model.layers.8.post_attention_layernorm.bias": 0.0,
149
+ "model.layers.9.self_attn.q_proj.weight": 6.007289243825653e-10,
150
+ "model.layers.9.self_attn.k_proj.weight": 6.41353707919734e-10,
151
+ "model.layers.9.self_attn.v_proj.weight": 2.956998343961459e-09,
152
+ "model.layers.9.self_attn.o_proj.weight": 2.4913035190365917e-09,
153
+ "model.layers.9.mlp.gate_proj.weight": 7.570661182398908e-09,
154
+ "model.layers.9.mlp.up_proj.weight": 6.353935380805827e-09,
155
+ "model.layers.9.mlp.down_proj.weight": 8.711306046886118e-10,
156
+ "model.layers.9.input_layernorm.weight": 0.0,
157
+ "model.layers.9.input_layernorm.bias": 2.168404344971009e-19,
158
+ "model.layers.9.post_attention_layernorm.weight": 0.0,
159
+ "model.layers.9.post_attention_layernorm.bias": 4.906014830496908e-19,
160
+ "model.layers.10.self_attn.q_proj.weight": 5.71690890183163e-10,
161
+ "model.layers.10.self_attn.k_proj.weight": 7.170714866333583e-10,
162
+ "model.layers.10.self_attn.v_proj.weight": 5.3668185273636486e-09,
163
+ "model.layers.10.self_attn.o_proj.weight": 1.271927203561063e-09,
164
+ "model.layers.10.mlp.gate_proj.weight": 5.268815736296043e-09,
165
+ "model.layers.10.mlp.up_proj.weight": 3.586305865610484e-09,
166
+ "model.layers.10.mlp.down_proj.weight": 5.071385089649993e-10,
167
+ "model.layers.10.input_layernorm.weight": 0.0,
168
+ "model.layers.10.input_layernorm.bias": 0.0,
169
+ "model.layers.10.post_attention_layernorm.weight": 0.0,
170
+ "model.layers.10.post_attention_layernorm.bias": 4.445228907190568e-19,
171
+ "model.layers.11.self_attn.q_proj.weight": 7.266194046451346e-10,
172
+ "model.layers.11.self_attn.k_proj.weight": 6.184255596508592e-10,
173
+ "model.layers.11.self_attn.v_proj.weight": 2.661776079548872e-09,
174
+ "model.layers.11.self_attn.o_proj.weight": 1.118204693284497e-09,
175
+ "model.layers.11.mlp.gate_proj.weight": 4.1432076614000836e-09,
176
+ "model.layers.11.mlp.up_proj.weight": 3.5073454303831873e-09,
177
+ "model.layers.11.mlp.down_proj.weight": 5.212785462542039e-10,
178
+ "model.layers.11.input_layernorm.weight": 0.0,
179
+ "model.layers.11.input_layernorm.bias": 0.0,
180
+ "model.layers.11.post_attention_layernorm.weight": 0.0,
181
+ "model.layers.11.post_attention_layernorm.bias": 2.168404344971009e-19,
182
+ "model.layers.12.self_attn.q_proj.weight": 5.733744146141361e-10,
183
+ "model.layers.12.self_attn.k_proj.weight": 5.487667209536085e-10,
184
+ "model.layers.12.self_attn.v_proj.weight": 4.031819287320104e-09,
185
+ "model.layers.12.self_attn.o_proj.weight": 1.7893317760808713e-09,
186
+ "model.layers.12.mlp.gate_proj.weight": 6.7238200548066134e-09,
187
+ "model.layers.12.mlp.up_proj.weight": 4.333610756172371e-09,
188
+ "model.layers.12.mlp.down_proj.weight": 3.8647418894773487e-10,
189
+ "model.layers.12.input_layernorm.weight": 0.0,
190
+ "model.layers.12.input_layernorm.bias": 1.7347234759768072e-19,
191
+ "model.layers.12.post_attention_layernorm.weight": 0.0,
192
+ "model.layers.12.post_attention_layernorm.bias": 1.7347234759768072e-19,
193
+ "model.layers.13.self_attn.q_proj.weight": 3.6206333930977054e-10,
194
+ "model.layers.13.self_attn.k_proj.weight": 3.510607982093461e-10,
195
+ "model.layers.13.self_attn.v_proj.weight": 3.862275548272009e-09,
196
+ "model.layers.13.self_attn.o_proj.weight": 8.800259365671081e-10,
197
+ "model.layers.13.mlp.gate_proj.weight": 5.46204698063472e-09,
198
+ "model.layers.13.mlp.up_proj.weight": 4.405306223513131e-09,
199
+ "model.layers.13.mlp.down_proj.weight": 4.107201808286239e-10,
200
+ "model.layers.13.input_layernorm.weight": 0.0,
201
+ "model.layers.13.input_layernorm.bias": 0.0,
202
+ "model.layers.13.post_attention_layernorm.weight": 0.0,
203
+ "model.layers.13.post_attention_layernorm.bias": 0.0,
204
+ "model.layers.14.self_attn.q_proj.weight": 3.8464165186269383e-10,
205
+ "model.layers.14.self_attn.k_proj.weight": 2.8596257806157154e-10,
206
+ "model.layers.14.self_attn.v_proj.weight": 3.1012530143925687e-09,
207
+ "model.layers.14.self_attn.o_proj.weight": 1.4942531834094552e-09,
208
+ "model.layers.14.mlp.gate_proj.weight": 6.4883253495888005e-09,
209
+ "model.layers.14.mlp.up_proj.weight": 6.394475349225104e-09,
210
+ "model.layers.14.mlp.down_proj.weight": 3.752011915973223e-10,
211
+ "model.layers.14.input_layernorm.weight": 0.0,
212
+ "model.layers.14.input_layernorm.bias": 0.0,
213
+ "model.layers.14.post_attention_layernorm.weight": 0.0,
214
+ "model.layers.14.post_attention_layernorm.bias": 4.0115480381963666e-19,
215
+ "model.layers.15.self_attn.q_proj.weight": 4.776023843078292e-10,
216
+ "model.layers.15.self_attn.k_proj.weight": 4.016206744950068e-10,
217
+ "model.layers.15.self_attn.v_proj.weight": 4.840656515625596e-09,
218
+ "model.layers.15.self_attn.o_proj.weight": 1.2165354235094129e-09,
219
+ "model.layers.15.mlp.gate_proj.weight": 6.1591943752622074e-09,
220
+ "model.layers.15.mlp.up_proj.weight": 5.179849722480867e-09,
221
+ "model.layers.15.mlp.down_proj.weight": 2.731740285829195e-10,
222
+ "model.layers.15.input_layernorm.weight": 0.0,
223
+ "model.layers.15.input_layernorm.bias": 9.75781955236954e-20,
224
+ "model.layers.15.post_attention_layernorm.weight": 0.0,
225
+ "model.layers.15.post_attention_layernorm.bias": 1.951563910473908e-19,
226
+ "model.layers.16.self_attn.q_proj.weight": 2.842091184618312e-10,
227
+ "model.layers.16.self_attn.k_proj.weight": 5.433628658124689e-10,
228
+ "model.layers.16.self_attn.v_proj.weight": 4.038230372316321e-09,
229
+ "model.layers.16.self_attn.o_proj.weight": 8.410473384401484e-10,
230
+ "model.layers.16.mlp.gate_proj.weight": 5.810029506392311e-09,
231
+ "model.layers.16.mlp.up_proj.weight": 4.040571184305008e-09,
232
+ "model.layers.16.mlp.down_proj.weight": 3.436520283154702e-10,
233
+ "model.layers.16.input_layernorm.weight": 0.0,
234
+ "model.layers.16.input_layernorm.bias": 4.336808689942018e-20,
235
+ "model.layers.16.post_attention_layernorm.weight": 0.0,
236
+ "model.layers.16.post_attention_layernorm.bias": 3.4694469519536144e-19,
237
+ "model.layers.17.self_attn.q_proj.weight": 5.428772098525769e-10,
238
+ "model.layers.17.self_attn.k_proj.weight": 4.891210991786466e-10,
239
+ "model.layers.17.self_attn.v_proj.weight": 5.218405476625776e-09,
240
+ "model.layers.17.self_attn.o_proj.weight": 1.6160302607204358e-09,
241
+ "model.layers.17.mlp.gate_proj.weight": 4.256941489681291e-09,
242
+ "model.layers.17.mlp.up_proj.weight": 2.7025096945484016e-09,
243
+ "model.layers.17.mlp.down_proj.weight": 3.0956850120370897e-10,
244
+ "model.layers.17.input_layernorm.weight": 0.0,
245
+ "model.layers.17.input_layernorm.bias": 0.0,
246
+ "model.layers.17.post_attention_layernorm.weight": 0.0,
247
+ "model.layers.17.post_attention_layernorm.bias": 2.2768245622195594e-19,
248
+ "model.layers.18.self_attn.q_proj.weight": 9.754786134408278e-10,
249
+ "model.layers.18.self_attn.k_proj.weight": 1.0905268510441602e-09,
250
+ "model.layers.18.self_attn.v_proj.weight": 1.7681153963167162e-09,
251
+ "model.layers.18.self_attn.o_proj.weight": 6.709172595265045e-10,
252
+ "model.layers.18.mlp.gate_proj.weight": 2.5347956504144045e-09,
253
+ "model.layers.18.mlp.up_proj.weight": 3.966439786988e-09,
254
+ "model.layers.18.mlp.down_proj.weight": 3.36461501622883e-10,
255
+ "model.layers.18.input_layernorm.weight": 0.0,
256
+ "model.layers.18.input_layernorm.bias": 1.7347234759768072e-19,
257
+ "model.layers.18.post_attention_layernorm.weight": 0.0,
258
+ "model.layers.18.post_attention_layernorm.bias": 1.0842021724855045e-20,
259
+ "model.layers.19.self_attn.q_proj.weight": 5.481194875756046e-10,
260
+ "model.layers.19.self_attn.k_proj.weight": 7.619642872214172e-10,
261
+ "model.layers.19.self_attn.v_proj.weight": 2.641356218191504e-09,
262
+ "model.layers.19.self_attn.o_proj.weight": 9.716190874087261e-10,
263
+ "model.layers.19.mlp.gate_proj.weight": 2.508716079319129e-09,
264
+ "model.layers.19.mlp.up_proj.weight": 2.292312615281844e-09,
265
+ "model.layers.19.mlp.down_proj.weight": 3.7359500974313657e-10,
266
+ "model.layers.19.input_layernorm.weight": 0.0,
267
+ "model.layers.19.input_layernorm.bias": 4.336808689942018e-20,
268
+ "model.layers.19.post_attention_layernorm.weight": 0.0,
269
+ "model.layers.19.post_attention_layernorm.bias": 3.496552006265752e-19,
270
+ "model.layers.20.self_attn.q_proj.weight": 4.460271441075747e-10,
271
+ "model.layers.20.self_attn.k_proj.weight": 5.93622040412356e-10,
272
+ "model.layers.20.self_attn.v_proj.weight": 2.9228209541543036e-09,
273
+ "model.layers.20.self_attn.o_proj.weight": 9.93879041288892e-10,
274
+ "model.layers.20.mlp.gate_proj.weight": 4.262360941235481e-09,
275
+ "model.layers.20.mlp.up_proj.weight": 2.8738960130188692e-09,
276
+ "model.layers.20.mlp.down_proj.weight": 3.880129639810548e-10,
277
+ "model.layers.20.input_layernorm.weight": 0.0,
278
+ "model.layers.20.input_layernorm.bias": 2.710505431213761e-19,
279
+ "model.layers.20.post_attention_layernorm.weight": 0.0,
280
+ "model.layers.20.post_attention_layernorm.bias": 3.686287386450715e-19,
281
+ "model.layers.21.self_attn.q_proj.weight": 7.124691592252929e-10,
282
+ "model.layers.21.self_attn.k_proj.weight": 5.755662257911353e-10,
283
+ "model.layers.21.self_attn.v_proj.weight": 1.891308869517161e-09,
284
+ "model.layers.21.self_attn.o_proj.weight": 1.4094457867486198e-09,
285
+ "model.layers.21.mlp.gate_proj.weight": 4.1152960269149236e-09,
286
+ "model.layers.21.mlp.up_proj.weight": 4.749576116106861e-09,
287
+ "model.layers.21.mlp.down_proj.weight": 6.98995326805137e-10,
288
+ "model.layers.21.input_layernorm.weight": 0.0,
289
+ "model.layers.21.input_layernorm.bias": 0.0,
290
+ "model.layers.21.post_attention_layernorm.weight": 0.0,
291
+ "model.layers.21.post_attention_layernorm.bias": 0.0,
292
+ "model.layers.22.self_attn.q_proj.weight": 9.83956525146823e-10,
293
+ "model.layers.22.self_attn.k_proj.weight": 7.087214726197998e-10,
294
+ "model.layers.22.self_attn.v_proj.weight": 2.4996680281219597e-09,
295
+ "model.layers.22.self_attn.o_proj.weight": 8.651306160345484e-10,
296
+ "model.layers.22.mlp.gate_proj.weight": 4.86394924337219e-09,
297
+ "model.layers.22.mlp.up_proj.weight": 4.5905771154745404e-09,
298
+ "model.layers.22.mlp.down_proj.weight": 3.95163937128018e-10,
299
+ "model.layers.22.input_layernorm.weight": 0.0,
300
+ "model.layers.22.input_layernorm.bias": 1.7347234759768072e-19,
301
+ "model.layers.22.post_attention_layernorm.weight": 0.0,
302
+ "model.layers.22.post_attention_layernorm.bias": 4.336808689942018e-20,
303
+ "model.layers.23.self_attn.q_proj.weight": 4.918045704016549e-10,
304
+ "model.layers.23.self_attn.k_proj.weight": 8.33777420439219e-10,
305
+ "model.layers.23.self_attn.v_proj.weight": 2.41253161448185e-09,
306
+ "model.layers.23.self_attn.o_proj.weight": 1.0251370241576296e-09,
307
+ "model.layers.23.mlp.gate_proj.weight": 7.800711803914358e-09,
308
+ "model.layers.23.mlp.up_proj.weight": 3.2208852947708994e-09,
309
+ "model.layers.23.mlp.down_proj.weight": 4.422955157679098e-10,
310
+ "model.layers.23.input_layernorm.weight": 0.0,
311
+ "model.layers.23.input_layernorm.bias": 3.9302328752599536e-19,
312
+ "model.layers.23.post_attention_layernorm.weight": 0.0,
313
+ "model.layers.23.post_attention_layernorm.bias": 0.0,
314
+ "model.layers.24.self_attn.q_proj.weight": 5.472487174529306e-10,
315
+ "model.layers.24.self_attn.k_proj.weight": 4.2753729445621503e-10,
316
+ "model.layers.24.self_attn.v_proj.weight": 1.443581112425818e-09,
317
+ "model.layers.24.self_attn.o_proj.weight": 8.208573376577987e-10,
318
+ "model.layers.24.mlp.gate_proj.weight": 2.998762624883966e-09,
319
+ "model.layers.24.mlp.up_proj.weight": 3.6555445603880797e-09,
320
+ "model.layers.24.mlp.down_proj.weight": 5.055675463457495e-10,
321
+ "model.layers.24.input_layernorm.weight": 0.0,
322
+ "model.layers.24.input_layernorm.bias": 0.0,
323
+ "model.layers.24.post_attention_layernorm.weight": 0.0,
324
+ "model.layers.24.post_attention_layernorm.bias": 0.0,
325
+ "model.layers.25.self_attn.q_proj.weight": 1.166278238429186e-09,
326
+ "model.layers.25.self_attn.k_proj.weight": 7.62426850542397e-10,
327
+ "model.layers.25.self_attn.v_proj.weight": 1.1932593224628363e-09,
328
+ "model.layers.25.self_attn.o_proj.weight": 8.748911994871378e-10,
329
+ "model.layers.25.mlp.gate_proj.weight": 5.748820512962994e-09,
330
+ "model.layers.25.mlp.up_proj.weight": 3.9719869467565635e-09,
331
+ "model.layers.25.mlp.down_proj.weight": 5.551282811211422e-10,
332
+ "model.layers.25.input_layernorm.weight": 0.0,
333
+ "model.layers.25.input_layernorm.bias": 4.336808689942018e-20,
334
+ "model.layers.25.post_attention_layernorm.weight": 0.0,
335
+ "model.layers.25.post_attention_layernorm.bias": 1.951563910473908e-19,
336
+ "model.layers.26.self_attn.q_proj.weight": 1.0714719422821872e-09,
337
+ "model.layers.26.self_attn.k_proj.weight": 6.015845599449676e-10,
338
+ "model.layers.26.self_attn.v_proj.weight": 8.382126992501071e-10,
339
+ "model.layers.26.self_attn.o_proj.weight": 6.916177852644978e-10,
340
+ "model.layers.26.mlp.gate_proj.weight": 5.836964949897568e-09,
341
+ "model.layers.26.mlp.up_proj.weight": 2.5183671444513798e-09,
342
+ "model.layers.26.mlp.down_proj.weight": 5.167603944755683e-10,
343
+ "model.layers.26.input_layernorm.weight": 0.0,
344
+ "model.layers.26.input_layernorm.bias": 0.0,
345
+ "model.layers.26.post_attention_layernorm.weight": 0.0,
346
+ "model.layers.26.post_attention_layernorm.bias": 0.0,
347
+ "model.layers.27.self_attn.q_proj.weight": 1.2824335726691061e-09,
348
+ "model.layers.27.self_attn.k_proj.weight": 5.505033939812165e-10,
349
+ "model.layers.27.self_attn.v_proj.weight": 1.0342314027411703e-09,
350
+ "model.layers.27.self_attn.o_proj.weight": 8.070220758327195e-10,
351
+ "model.layers.27.mlp.gate_proj.weight": 9.268433132092468e-09,
352
+ "model.layers.27.mlp.up_proj.weight": 4.086552962689893e-09,
353
+ "model.layers.27.mlp.down_proj.weight": 5.494656344732599e-10,
354
+ "model.layers.27.input_layernorm.weight": 0.0,
355
+ "model.layers.27.input_layernorm.bias": 0.0,
356
+ "model.layers.27.post_attention_layernorm.weight": 0.0,
357
+ "model.layers.27.post_attention_layernorm.bias": 4.336808689942018e-20,
358
+ "model.layers.28.self_attn.q_proj.weight": 1.4427759253976547e-09,
359
+ "model.layers.28.self_attn.k_proj.weight": 8.499876003043027e-10,
360
+ "model.layers.28.self_attn.v_proj.weight": 2.8016114583806485e-09,
361
+ "model.layers.28.self_attn.o_proj.weight": 8.107200244467094e-10,
362
+ "model.layers.28.mlp.gate_proj.weight": 5.27831199785093e-09,
363
+ "model.layers.28.mlp.up_proj.weight": 5.316902994915533e-09,
364
+ "model.layers.28.mlp.down_proj.weight": 5.632971541065975e-10,
365
+ "model.layers.28.input_layernorm.weight": 0.0,
366
+ "model.layers.28.input_layernorm.bias": 3.4846935450041914e-19,
367
+ "model.layers.28.post_attention_layernorm.weight": 0.0,
368
+ "model.layers.28.post_attention_layernorm.bias": 0.0,
369
+ "model.layers.29.self_attn.q_proj.weight": 9.260426736545924e-10,
370
+ "model.layers.29.self_attn.k_proj.weight": 6.835472277089139e-10,
371
+ "model.layers.29.self_attn.v_proj.weight": 9.57920534006007e-10,
372
+ "model.layers.29.self_attn.o_proj.weight": 9.050408777966367e-10,
373
+ "model.layers.29.mlp.gate_proj.weight": 7.60078516274613e-09,
374
+ "model.layers.29.mlp.up_proj.weight": 3.756985430906449e-09,
375
+ "model.layers.29.mlp.down_proj.weight": 4.79516586437967e-10,
376
+ "model.layers.29.input_layernorm.weight": 0.0,
377
+ "model.layers.29.input_layernorm.bias": 0.0,
378
+ "model.layers.29.post_attention_layernorm.weight": 0.0,
379
+ "model.layers.29.post_attention_layernorm.bias": 0.0,
380
+ "model.layers.30.self_attn.q_proj.weight": 1.3636689288887282e-09,
381
+ "model.layers.30.self_attn.k_proj.weight": 1.5714329038019058e-09,
382
+ "model.layers.30.self_attn.v_proj.weight": 1.1758381646131966e-09,
383
+ "model.layers.30.self_attn.o_proj.weight": 9.795578392868266e-10,
384
+ "model.layers.30.mlp.gate_proj.weight": 9.374549942246327e-09,
385
+ "model.layers.30.mlp.up_proj.weight": 5.656590928992955e-09,
386
+ "model.layers.30.mlp.down_proj.weight": 5.065968859222873e-10,
387
+ "model.layers.30.input_layernorm.weight": 0.0,
388
+ "model.layers.30.input_layernorm.bias": 0.0,
389
+ "model.layers.30.post_attention_layernorm.weight": 0.0,
390
+ "model.layers.30.post_attention_layernorm.bias": 0.0,
391
+ "model.layers.31.self_attn.q_proj.weight": 2.28110124567138e-09,
392
+ "model.layers.31.self_attn.k_proj.weight": 1.3092864037389517e-09,
393
+ "model.layers.31.self_attn.v_proj.weight": 1.0963311325440373e-09,
394
+ "model.layers.31.self_attn.o_proj.weight": 9.979353166045257e-10,
395
+ "model.layers.31.mlp.gate_proj.weight": 1.401771745198251e-08,
396
+ "model.layers.31.mlp.up_proj.weight": 4.416380458375594e-09,
397
+ "model.layers.31.mlp.down_proj.weight": 7.472823426724062e-10,
398
+ "model.layers.31.input_layernorm.weight": 0.0,
399
+ "model.layers.31.input_layernorm.bias": 1.7347234759768072e-19,
400
+ "model.layers.31.post_attention_layernorm.weight": 0.0,
401
+ "model.layers.31.post_attention_layernorm.bias": 1.7347234759768072e-19,
402
+ "model.layers.32.self_attn.q_proj.weight": 1.4159746797304251e-09,
403
+ "model.layers.32.self_attn.k_proj.weight": 1.0075619627514242e-09,
404
+ "model.layers.32.self_attn.v_proj.weight": 2.012204163293063e-09,
405
+ "model.layers.32.self_attn.o_proj.weight": 1.2717424624497653e-09,
406
+ "model.layers.32.mlp.gate_proj.weight": 1.0099245173478266e-08,
407
+ "model.layers.32.mlp.up_proj.weight": 9.207220349101893e-09,
408
+ "model.layers.32.mlp.down_proj.weight": 8.522503473310887e-10,
409
+ "model.layers.32.input_layernorm.weight": 0.0,
410
+ "model.layers.32.input_layernorm.bias": 0.0,
411
+ "model.layers.32.post_attention_layernorm.weight": 0.0,
412
+ "model.layers.32.post_attention_layernorm.bias": 0.0,
413
+ "model.layers.33.self_attn.q_proj.weight": 1.7309234578988253e-09,
414
+ "model.layers.33.self_attn.k_proj.weight": 1.6029864013944461e-09,
415
+ "model.layers.33.self_attn.v_proj.weight": 4.697651831975236e-09,
416
+ "model.layers.33.self_attn.o_proj.weight": 1.531523423636827e-09,
417
+ "model.layers.33.mlp.gate_proj.weight": 1.4452407413045876e-08,
418
+ "model.layers.33.mlp.up_proj.weight": 9.082494519437507e-09,
419
+ "model.layers.33.mlp.down_proj.weight": 9.669621666337964e-10,
420
+ "model.layers.33.input_layernorm.weight": 0.0,
421
+ "model.layers.33.input_layernorm.bias": 4.336808689942018e-20,
422
+ "model.layers.33.post_attention_layernorm.weight": 0.0,
423
+ "model.layers.33.post_attention_layernorm.bias": 0.0,
424
+ "model.layers.34.self_attn.q_proj.weight": 2.4314635993505364e-09,
425
+ "model.layers.34.self_attn.k_proj.weight": 1.703990619716933e-09,
426
+ "model.layers.34.self_attn.v_proj.weight": 2.379507009209192e-09,
427
+ "model.layers.34.self_attn.o_proj.weight": 1.3013656996463397e-09,
428
+ "model.layers.34.mlp.gate_proj.weight": 1.7839614277666746e-08,
429
+ "model.layers.34.mlp.up_proj.weight": 8.645561896022022e-09,
430
+ "model.layers.34.mlp.down_proj.weight": 7.185560946254554e-10,
431
+ "model.layers.34.input_layernorm.weight": 0.0,
432
+ "model.layers.34.input_layernorm.bias": 0.0,
433
+ "model.layers.34.post_attention_layernorm.weight": 0.0,
434
+ "model.layers.34.post_attention_layernorm.bias": 0.0,
435
+ "model.layers.35.self_attn.q_proj.weight": 2.9998821560184297e-09,
436
+ "model.layers.35.self_attn.k_proj.weight": 9.999101990842974e-10,
437
+ "model.layers.35.self_attn.v_proj.weight": 3.91424379131422e-09,
438
+ "model.layers.35.self_attn.o_proj.weight": 1.8113006206021964e-09,
439
+ "model.layers.35.mlp.gate_proj.weight": 2.1299209341426224e-08,
440
+ "model.layers.35.mlp.up_proj.weight": 1.5169396950417043e-08,
441
+ "model.layers.35.mlp.down_proj.weight": 2.0139570248526678e-09,
442
+ "model.layers.35.input_layernorm.weight": 0.0,
443
+ "model.layers.35.input_layernorm.bias": 1.7347234759768072e-19,
444
+ "model.layers.35.post_attention_layernorm.weight": 0.0,
445
+ "model.layers.35.post_attention_layernorm.bias": 0.0,
446
+ "model.layers.36.self_attn.q_proj.weight": 2.6555719045973093e-09,
447
+ "model.layers.36.self_attn.k_proj.weight": 1.5296728861358134e-09,
448
+ "model.layers.36.self_attn.v_proj.weight": 6.2084450291877144e-09,
449
+ "model.layers.36.self_attn.o_proj.weight": 1.9505691284393834e-09,
450
+ "model.layers.36.mlp.gate_proj.weight": 1.6187862759882893e-08,
451
+ "model.layers.36.mlp.up_proj.weight": 1.652175493897327e-08,
452
+ "model.layers.36.mlp.down_proj.weight": 1.3528854954832544e-09,
453
+ "model.layers.36.input_layernorm.weight": 0.0,
454
+ "model.layers.36.input_layernorm.bias": 0.0,
455
+ "model.layers.36.post_attention_layernorm.weight": 0.0,
456
+ "model.layers.36.post_attention_layernorm.bias": 1.7347234759768072e-19,
457
+ "model.layers.37.self_attn.q_proj.weight": 2.406661110399e-09,
458
+ "model.layers.37.self_attn.k_proj.weight": 1.7653792383498513e-09,
459
+ "model.layers.37.self_attn.v_proj.weight": 7.7608228821191e-09,
460
+ "model.layers.37.self_attn.o_proj.weight": 2.089813762040649e-09,
461
+ "model.layers.37.mlp.gate_proj.weight": 1.0708132928508955e-08,
462
+ "model.layers.37.mlp.up_proj.weight": 9.600955763744423e-09,
463
+ "model.layers.37.mlp.down_proj.weight": 1.129514638857169e-09,
464
+ "model.layers.37.input_layernorm.weight": 0.0,
465
+ "model.layers.37.input_layernorm.bias": 0.0,
466
+ "model.layers.37.post_attention_layernorm.weight": 0.0,
467
+ "model.layers.37.post_attention_layernorm.bias": 0.0,
468
+ "model.layers.38.self_attn.q_proj.weight": 2.8534219609355205e-09,
469
+ "model.layers.38.self_attn.k_proj.weight": 2.46495147848691e-09,
470
+ "model.layers.38.self_attn.v_proj.weight": 5.143008934282989e-09,
471
+ "model.layers.38.self_attn.o_proj.weight": 2.608180125207582e-09,
472
+ "model.layers.38.mlp.gate_proj.weight": 1.637199299390583e-08,
473
+ "model.layers.38.mlp.up_proj.weight": 7.963470428270133e-09,
474
+ "model.layers.38.mlp.down_proj.weight": 2.3918623052547144e-09,
475
+ "model.layers.38.input_layernorm.weight": 0.0,
476
+ "model.layers.38.input_layernorm.bias": 0.0,
477
+ "model.layers.38.post_attention_layernorm.weight": 0.0,
478
+ "model.layers.38.post_attention_layernorm.bias": 0.0,
479
+ "model.layers.39.self_attn.q_proj.weight": 1.945519016999242e-09,
480
+ "model.layers.39.self_attn.k_proj.weight": 1.2198944432384451e-09,
481
+ "model.layers.39.self_attn.v_proj.weight": 5.185719373912434e-09,
482
+ "model.layers.39.self_attn.o_proj.weight": 8.505105597578222e-09,
483
+ "model.layers.39.mlp.gate_proj.weight": 2.532518919906579e-08,
484
+ "model.layers.39.mlp.up_proj.weight": 1.6030430742830504e-08,
485
+ "model.layers.39.mlp.down_proj.weight": 2.789723036282036e-09,
486
+ "model.layers.39.input_layernorm.weight": 0.0,
487
+ "model.layers.39.input_layernorm.bias": 0.0,
488
+ "model.layers.39.post_attention_layernorm.weight": 0.0,
489
+ "model.layers.39.post_attention_layernorm.bias": 0.0,
490
+ "model.norm.weight": 0.0,
491
+ "model.norm.bias": 0.0,
492
+ "lm_head.weight": 1.5311139267753199e-19
493
+ },
494
+ "module_to_params": {
495
+ "model.embed_tokens.weight": 2.917434234714228e-19,
496
+ "model.layers.0.self_attn": 7.920487519186281e-09,
497
+ "model.layers.0.mlp": 1.1966177832315832e-08,
498
+ "model.layers.0.input_layernorm": 2.710505431213761e-19,
499
+ "model.layers.0.post_attention_layernorm": 6.938893903907229e-19,
500
+ "model.layers.1.self_attn": 2.2341763070699017e-08,
501
+ "model.layers.1.mlp": 4.642010935640428e-09,
502
+ "model.layers.1.input_layernorm": 3.9916427639358903e-19,
503
+ "model.layers.1.post_attention_layernorm": 1.8431436932253575e-19,
504
+ "model.layers.2.self_attn": 9.60992252885262e-09,
505
+ "model.layers.2.mlp": 3.582362779752657e-09,
506
+ "model.layers.2.input_layernorm": 2.9544509200229997e-19,
507
+ "model.layers.2.post_attention_layernorm": 4.336808689942018e-20,
508
+ "model.layers.3.self_attn": 8.925772405632415e-09,
509
+ "model.layers.3.mlp": 5.664624135685396e-08,
510
+ "model.layers.3.input_layernorm": 1.328147661294743e-19,
511
+ "model.layers.3.post_attention_layernorm": 1.9786689647860455e-19,
512
+ "model.layers.4.self_attn": 7.112303066492131e-09,
513
+ "model.layers.4.mlp": 3.493962632698337e-08,
514
+ "model.layers.4.input_layernorm": 4.797594613248357e-19,
515
+ "model.layers.4.post_attention_layernorm": 1.8702487475374952e-19,
516
+ "model.layers.5.self_attn": 6.413531963289643e-09,
517
+ "model.layers.5.mlp": 7.511871444876305e-09,
518
+ "model.layers.5.input_layernorm": 0.0,
519
+ "model.layers.5.post_attention_layernorm": 4.445228907190568e-19,
520
+ "model.layers.6.self_attn": 5.3933323940214e-09,
521
+ "model.layers.6.mlp": 7.4314101065435046e-09,
522
+ "model.layers.6.input_layernorm": 0.0,
523
+ "model.layers.6.post_attention_layernorm": 2.2768245622195594e-19,
524
+ "model.layers.7.self_attn": 6.313910247968123e-09,
525
+ "model.layers.7.mlp": 5.013754484176995e-09,
526
+ "model.layers.7.input_layernorm": 1.7618285302889447e-19,
527
+ "model.layers.7.post_attention_layernorm": 1.2197274440461925e-19,
528
+ "model.layers.8.self_attn": 6.397113736511528e-09,
529
+ "model.layers.8.mlp": 7.55865182592667e-09,
530
+ "model.layers.8.input_layernorm": 4.336808689942018e-20,
531
+ "model.layers.8.post_attention_layernorm": 0.0,
532
+ "model.layers.9.self_attn": 6.690384495300349e-09,
533
+ "model.layers.9.mlp": 1.4795727167893346e-08,
534
+ "model.layers.9.input_layernorm": 2.168404344971009e-19,
535
+ "model.layers.9.post_attention_layernorm": 4.906014830496908e-19,
536
+ "model.layers.10.self_attn": 7.927508107741233e-09,
537
+ "model.layers.10.mlp": 9.362260110871526e-09,
538
+ "model.layers.10.input_layernorm": 0.0,
539
+ "model.layers.10.post_attention_layernorm": 4.445228907190568e-19,
540
+ "model.layers.11.self_attn": 5.125025737129363e-09,
541
+ "model.layers.11.mlp": 8.171831638037475e-09,
542
+ "model.layers.11.input_layernorm": 0.0,
543
+ "model.layers.11.post_attention_layernorm": 2.168404344971009e-19,
544
+ "model.layers.12.self_attn": 6.943292198968721e-09,
545
+ "model.layers.12.mlp": 1.1443904999926721e-08,
546
+ "model.layers.12.input_layernorm": 1.7347234759768072e-19,
547
+ "model.layers.12.post_attention_layernorm": 1.7347234759768072e-19,
548
+ "model.layers.13.self_attn": 5.4554256223582344e-09,
549
+ "model.layers.13.mlp": 1.0278073384976474e-08,
550
+ "model.layers.13.input_layernorm": 0.0,
551
+ "model.layers.13.post_attention_layernorm": 0.0,
552
+ "model.layers.14.self_attn": 5.2661104277262895e-09,
553
+ "model.layers.14.mlp": 1.3258001890411226e-08,
554
+ "model.layers.14.input_layernorm": 0.0,
555
+ "model.layers.14.post_attention_layernorm": 4.0115480381963666e-19,
556
+ "model.layers.15.self_attn": 6.936414997937845e-09,
557
+ "model.layers.15.mlp": 1.1612218126325994e-08,
558
+ "model.layers.15.input_layernorm": 9.75781955236954e-20,
559
+ "model.layers.15.post_attention_layernorm": 1.951563910473908e-19,
560
+ "model.layers.16.self_attn": 5.706849695030769e-09,
561
+ "model.layers.16.mlp": 1.0194252719012789e-08,
562
+ "model.layers.16.input_layernorm": 4.336808689942018e-20,
563
+ "model.layers.16.post_attention_layernorm": 3.4694469519536144e-19,
564
+ "model.layers.17.self_attn": 7.866434046377435e-09,
565
+ "model.layers.17.mlp": 7.269019685433401e-09,
566
+ "model.layers.17.input_layernorm": 0.0,
567
+ "model.layers.17.post_attention_layernorm": 2.2768245622195594e-19,
568
+ "model.layers.18.self_attn": 4.505038120328209e-09,
569
+ "model.layers.18.mlp": 6.837696939025288e-09,
570
+ "model.layers.18.input_layernorm": 1.7347234759768072e-19,
571
+ "model.layers.18.post_attention_layernorm": 1.0842021724855045e-20,
572
+ "model.layers.19.self_attn": 4.923059080397252e-09,
573
+ "model.layers.19.mlp": 5.174623704344109e-09,
574
+ "model.layers.19.input_layernorm": 4.336808689942018e-20,
575
+ "model.layers.19.post_attention_layernorm": 3.496552006265752e-19,
576
+ "model.layers.20.self_attn": 4.9563491799631266e-09,
577
+ "model.layers.20.mlp": 7.524269918235405e-09,
578
+ "model.layers.20.input_layernorm": 2.710505431213761e-19,
579
+ "model.layers.20.post_attention_layernorm": 3.686287386450715e-19,
580
+ "model.layers.21.self_attn": 4.5887900412822085e-09,
581
+ "model.layers.21.mlp": 9.56386746982692e-09,
582
+ "model.layers.21.input_layernorm": 0.0,
583
+ "model.layers.21.post_attention_layernorm": 0.0,
584
+ "model.layers.22.self_attn": 5.057476641923131e-09,
585
+ "model.layers.22.mlp": 9.849690295974748e-09,
586
+ "model.layers.22.input_layernorm": 1.7347234759768072e-19,
587
+ "model.layers.22.post_attention_layernorm": 4.336808689942018e-20,
588
+ "model.layers.23.self_attn": 4.763250629480353e-09,
589
+ "model.layers.23.mlp": 1.1463892614453168e-08,
590
+ "model.layers.23.input_layernorm": 3.9302328752599536e-19,
591
+ "model.layers.23.post_attention_layernorm": 0.0,
592
+ "model.layers.24.self_attn": 3.239224461992763e-09,
593
+ "model.layers.24.mlp": 7.159874731617795e-09,
594
+ "model.layers.24.input_layernorm": 0.0,
595
+ "model.layers.24.post_attention_layernorm": 0.0,
596
+ "model.layers.25.self_attn": 3.996855610921557e-09,
597
+ "model.layers.25.mlp": 1.0275935740840701e-08,
598
+ "model.layers.25.input_layernorm": 4.336808689942018e-20,
599
+ "model.layers.25.post_attention_layernorm": 1.951563910473908e-19,
600
+ "model.layers.26.self_attn": 3.2028869867417595e-09,
601
+ "model.layers.26.mlp": 8.872092488824517e-09,
602
+ "model.layers.26.input_layernorm": 0.0,
603
+ "model.layers.26.post_attention_layernorm": 0.0,
604
+ "model.layers.27.self_attn": 3.6741904452242126e-09,
605
+ "model.layers.27.mlp": 1.3904451729255621e-08,
606
+ "model.layers.27.input_layernorm": 0.0,
607
+ "model.layers.27.post_attention_layernorm": 4.336808689942018e-20,
608
+ "model.layers.28.self_attn": 5.905095008529315e-09,
609
+ "model.layers.28.mlp": 1.115851214687306e-08,
610
+ "model.layers.28.input_layernorm": 3.4846935450041914e-19,
611
+ "model.layers.28.post_attention_layernorm": 0.0,
612
+ "model.layers.29.self_attn": 3.47255131316615e-09,
613
+ "model.layers.29.mlp": 1.1837287180090546e-08,
614
+ "model.layers.29.input_layernorm": 0.0,
615
+ "model.layers.29.post_attention_layernorm": 0.0,
616
+ "model.layers.30.self_attn": 5.090497836590657e-09,
617
+ "model.layers.30.mlp": 1.553773775716157e-08,
618
+ "model.layers.30.input_layernorm": 0.0,
619
+ "model.layers.30.post_attention_layernorm": 0.0,
620
+ "model.layers.31.self_attn": 5.684654098558895e-09,
621
+ "model.layers.31.mlp": 1.9181380253030513e-08,
622
+ "model.layers.31.input_layernorm": 1.7347234759768072e-19,
623
+ "model.layers.31.post_attention_layernorm": 1.7347234759768072e-19,
624
+ "model.layers.32.self_attn": 5.707483268224678e-09,
625
+ "model.layers.32.mlp": 2.0158715869911248e-08,
626
+ "model.layers.32.input_layernorm": 0.0,
627
+ "model.layers.32.post_attention_layernorm": 0.0,
628
+ "model.layers.33.self_attn": 9.563085114905333e-09,
629
+ "model.layers.33.mlp": 2.4501864099117178e-08,
630
+ "model.layers.33.input_layernorm": 4.336808689942018e-20,
631
+ "model.layers.33.post_attention_layernorm": 0.0,
632
+ "model.layers.34.self_attn": 7.816326927923e-09,
633
+ "model.layers.34.mlp": 2.7203732268314224e-08,
634
+ "model.layers.34.input_layernorm": 0.0,
635
+ "model.layers.34.post_attention_layernorm": 0.0,
636
+ "model.layers.35.self_attn": 9.725336767019142e-09,
637
+ "model.layers.35.mlp": 3.848256331669593e-08,
638
+ "model.layers.35.input_layernorm": 1.7347234759768072e-19,
639
+ "model.layers.35.post_attention_layernorm": 0.0,
640
+ "model.layers.36.self_attn": 1.234425894836022e-08,
641
+ "model.layers.36.mlp": 3.406250319433942e-08,
642
+ "model.layers.36.input_layernorm": 0.0,
643
+ "model.layers.36.post_attention_layernorm": 1.7347234759768072e-19,
644
+ "model.layers.37.self_attn": 1.40226769929086e-08,
645
+ "model.layers.37.mlp": 2.1438603331110546e-08,
646
+ "model.layers.37.input_layernorm": 0.0,
647
+ "model.layers.37.post_attention_layernorm": 0.0,
648
+ "model.layers.38.self_attn": 1.3069562498913002e-08,
649
+ "model.layers.38.mlp": 2.6727325727430677e-08,
650
+ "model.layers.38.input_layernorm": 0.0,
651
+ "model.layers.38.post_attention_layernorm": 0.0,
652
+ "model.layers.39.self_attn": 1.6856238431728344e-08,
653
+ "model.layers.39.mlp": 4.414534297817833e-08,
654
+ "model.layers.39.input_layernorm": 0.0,
655
+ "model.layers.39.post_attention_layernorm": 0.0,
656
+ "model.norm.weight": 0.0,
657
+ "model.norm.bias": 0.0,
658
+ "lm_head.weight": 1.5311139267753199e-19
659
+ }
660
+ }
analysis.json ADDED
@@ -0,0 +1,660 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "aveage_var_degree": 3.892243738108603e-09,
3
+ "total_diff_num": 11385304937,
4
+ "total_param_num": 14498703360,
5
+ "diff_rate": 78.5263664915757,
6
+ "layer_to_params": {
7
+ "0": 1.988666535246705e-08,
8
+ "1": 2.6983774006922925e-08,
9
+ "2": 1.319228530894409e-08,
10
+ "3": 6.557201376281705e-08,
11
+ "4": 4.205192939414228e-08,
12
+ "5": 1.3925403408610472e-08,
13
+ "6": 1.2824742500792588e-08,
14
+ "7": 1.1327664732443272e-08,
15
+ "8": 1.3955765562481565e-08,
16
+ "9": 2.148611166390114e-08,
17
+ "10": 1.7289768219057282e-08,
18
+ "11": 1.329685737538368e-08,
19
+ "12": 1.8387197199242386e-08,
20
+ "13": 1.573349900733471e-08,
21
+ "14": 1.8524112318538675e-08,
22
+ "15": 1.8548633124556574e-08,
23
+ "16": 1.590110241443387e-08,
24
+ "17": 1.5135453732038522e-08,
25
+ "18": 1.1342735059537811e-08,
26
+ "19": 1.0097682785134383e-08,
27
+ "20": 1.248061909883821e-08,
28
+ "21": 1.415265751110913e-08,
29
+ "22": 1.490716693811472e-08,
30
+ "23": 1.6227143244326544e-08,
31
+ "24": 1.0399099193610557e-08,
32
+ "25": 1.4272791352000781e-08,
33
+ "26": 1.2074979475566277e-08,
34
+ "27": 1.75786421745232e-08,
35
+ "28": 1.7063607155750844e-08,
36
+ "29": 1.5309838493256697e-08,
37
+ "30": 2.0628235593752226e-08,
38
+ "31": 2.4866034351936353e-08,
39
+ "32": 2.5866199138135922e-08,
40
+ "33": 3.406494921406588e-08,
41
+ "34": 3.5020059196237226e-08,
42
+ "35": 4.820790008388855e-08,
43
+ "36": 4.640676214287311e-08,
44
+ "37": 3.5461280324019145e-08,
45
+ "38": 3.979688822634368e-08,
46
+ "39": 6.100158140990667e-08
47
+ },
48
+ "key_to_params": {
49
+ "model.embed_tokens.weight": 2.917434234714228e-19,
50
+ "model.layers.0.self_attn.q_proj.weight": 1.187384697232119e-09,
51
+ "model.layers.0.self_attn.k_proj.weight": 1.1349608541877388e-09,
52
+ "model.layers.0.self_attn.v_proj.weight": 3.0414742013817886e-09,
53
+ "model.layers.0.self_attn.o_proj.weight": 2.556667766384635e-09,
54
+ "model.layers.0.mlp.gate_proj.weight": 8.003662514965982e-09,
55
+ "model.layers.0.mlp.up_proj.weight": 2.3990156705622212e-09,
56
+ "model.layers.0.mlp.down_proj.weight": 1.5634996467876287e-09,
57
+ "model.layers.0.input_layernorm.weight": 0.0,
58
+ "model.layers.0.input_layernorm.bias": 2.710505431213761e-19,
59
+ "model.layers.0.post_attention_layernorm.weight": 0.0,
60
+ "model.layers.0.post_attention_layernorm.bias": 6.938893903907229e-19,
61
+ "model.layers.1.self_attn.q_proj.weight": 2.0518091048415955e-09,
62
+ "model.layers.1.self_attn.k_proj.weight": 1.6146566395036643e-09,
63
+ "model.layers.1.self_attn.v_proj.weight": 1.4650281627837103e-08,
64
+ "model.layers.1.self_attn.o_proj.weight": 4.025015698516654e-09,
65
+ "model.layers.1.mlp.gate_proj.weight": 1.7941124497156125e-09,
66
+ "model.layers.1.mlp.up_proj.weight": 2.191004947841672e-09,
67
+ "model.layers.1.mlp.down_proj.weight": 6.568935380831438e-10,
68
+ "model.layers.1.input_layernorm.weight": 0.0,
69
+ "model.layers.1.input_layernorm.bias": 3.9916427639358903e-19,
70
+ "model.layers.1.post_attention_layernorm.weight": 0.0,
71
+ "model.layers.1.post_attention_layernorm.bias": 1.8431436932253575e-19,
72
+ "model.layers.2.self_attn.q_proj.weight": 8.317828559256668e-10,
73
+ "model.layers.2.self_attn.k_proj.weight": 1.0357459956367166e-09,
74
+ "model.layers.2.self_attn.v_proj.weight": 4.803724777957541e-09,
75
+ "model.layers.2.self_attn.o_proj.weight": 2.938668899332697e-09,
76
+ "model.layers.2.mlp.gate_proj.weight": 1.4420785040177482e-09,
77
+ "model.layers.2.mlp.up_proj.weight": 1.6792421320133145e-09,
78
+ "model.layers.2.mlp.down_proj.weight": 4.6104214372159426e-10,
79
+ "model.layers.2.input_layernorm.weight": 0.0,
80
+ "model.layers.2.input_layernorm.bias": 2.9544509200229997e-19,
81
+ "model.layers.2.post_attention_layernorm.weight": 0.0,
82
+ "model.layers.2.post_attention_layernorm.bias": 4.336808689942018e-20,
83
+ "model.layers.3.self_attn.q_proj.weight": 8.837429277264163e-10,
84
+ "model.layers.3.self_attn.k_proj.weight": 1.0162795405221914e-09,
85
+ "model.layers.3.self_attn.v_proj.weight": 3.0072123990976253e-09,
86
+ "model.layers.3.self_attn.o_proj.weight": 4.018537538286182e-09,
87
+ "model.layers.3.mlp.gate_proj.weight": 1.4122169507876e-09,
88
+ "model.layers.3.mlp.up_proj.weight": 1.5453091843179814e-09,
89
+ "model.layers.3.mlp.down_proj.weight": 5.368871522174838e-08,
90
+ "model.layers.3.input_layernorm.weight": 0.0,
91
+ "model.layers.3.input_layernorm.bias": 1.328147661294743e-19,
92
+ "model.layers.3.post_attention_layernorm.weight": 0.0,
93
+ "model.layers.3.post_attention_layernorm.bias": 1.9786689647860455e-19,
94
+ "model.layers.4.self_attn.q_proj.weight": 8.681465857307558e-10,
95
+ "model.layers.4.self_attn.k_proj.weight": 1.7090624737647886e-09,
96
+ "model.layers.4.self_attn.v_proj.weight": 2.7331961405252512e-09,
97
+ "model.layers.4.self_attn.o_proj.weight": 1.8018978664713358e-09,
98
+ "model.layers.4.mlp.gate_proj.weight": 1.7898181188987412e-09,
99
+ "model.layers.4.mlp.up_proj.weight": 1.897217695538226e-09,
100
+ "model.layers.4.mlp.down_proj.weight": 3.12525905125464e-08,
101
+ "model.layers.4.input_layernorm.weight": 0.0,
102
+ "model.layers.4.input_layernorm.bias": 4.797594613248357e-19,
103
+ "model.layers.4.post_attention_layernorm.weight": 0.0,
104
+ "model.layers.4.post_attention_layernorm.bias": 1.8702487475374952e-19,
105
+ "model.layers.5.self_attn.q_proj.weight": 9.632125852476747e-10,
106
+ "model.layers.5.self_attn.k_proj.weight": 7.343454910824221e-10,
107
+ "model.layers.5.self_attn.v_proj.weight": 3.0922541327527143e-09,
108
+ "model.layers.5.self_attn.o_proj.weight": 1.6237197542068315e-09,
109
+ "model.layers.5.mlp.gate_proj.weight": 1.7408918514168667e-09,
110
+ "model.layers.5.mlp.up_proj.weight": 1.8568865319442314e-09,
111
+ "model.layers.5.mlp.down_proj.weight": 3.914093061515208e-09,
112
+ "model.layers.5.input_layernorm.weight": 0.0,
113
+ "model.layers.5.input_layernorm.bias": 0.0,
114
+ "model.layers.5.post_attention_layernorm.weight": 0.0,
115
+ "model.layers.5.post_attention_layernorm.bias": 4.445228907190568e-19,
116
+ "model.layers.6.self_attn.q_proj.weight": 4.4907185525744355e-10,
117
+ "model.layers.6.self_attn.k_proj.weight": 6.927900386699548e-10,
118
+ "model.layers.6.self_attn.v_proj.weight": 2.1942298644717086e-09,
119
+ "model.layers.6.self_attn.o_proj.weight": 2.0572406356222928e-09,
120
+ "model.layers.6.mlp.gate_proj.weight": 3.026063382094435e-09,
121
+ "model.layers.6.mlp.up_proj.weight": 3.793664973272826e-09,
122
+ "model.layers.6.mlp.down_proj.weight": 6.116817511762444e-10,
123
+ "model.layers.6.input_layernorm.weight": 0.0,
124
+ "model.layers.6.input_layernorm.bias": 0.0,
125
+ "model.layers.6.post_attention_layernorm.weight": 0.0,
126
+ "model.layers.6.post_attention_layernorm.bias": 2.2768245622195594e-19,
127
+ "model.layers.7.self_attn.q_proj.weight": 5.192902108319686e-10,
128
+ "model.layers.7.self_attn.k_proj.weight": 4.908492456934255e-10,
129
+ "model.layers.7.self_attn.v_proj.weight": 3.6511215739665203e-09,
130
+ "model.layers.7.self_attn.o_proj.weight": 1.6526492174762098e-09,
131
+ "model.layers.7.mlp.gate_proj.weight": 1.8395443627620504e-09,
132
+ "model.layers.7.mlp.up_proj.weight": 2.646799543981615e-09,
133
+ "model.layers.7.mlp.down_proj.weight": 5.274105774333293e-10,
134
+ "model.layers.7.input_layernorm.weight": 0.0,
135
+ "model.layers.7.input_layernorm.bias": 1.7618285302889447e-19,
136
+ "model.layers.7.post_attention_layernorm.weight": 0.0,
137
+ "model.layers.7.post_attention_layernorm.bias": 1.2197274440461925e-19,
138
+ "model.layers.8.self_attn.q_proj.weight": 6.803252006193361e-10,
139
+ "model.layers.8.self_attn.k_proj.weight": 1.0779841375097021e-09,
140
+ "model.layers.8.self_attn.v_proj.weight": 2.402063898898632e-09,
141
+ "model.layers.8.self_attn.o_proj.weight": 2.236740499483858e-09,
142
+ "model.layers.8.mlp.gate_proj.weight": 4.132192543693236e-09,
143
+ "model.layers.8.mlp.up_proj.weight": 2.8210202647945454e-09,
144
+ "model.layers.8.mlp.down_proj.weight": 6.054390174388875e-10,
145
+ "model.layers.8.input_layernorm.weight": 0.0,
146
+ "model.layers.8.input_layernorm.bias": 4.336808689942018e-20,
147
+ "model.layers.8.post_attention_layernorm.weight": 0.0,
148
+ "model.layers.8.post_attention_layernorm.bias": 0.0,
149
+ "model.layers.9.self_attn.q_proj.weight": 6.007289243825653e-10,
150
+ "model.layers.9.self_attn.k_proj.weight": 6.41353707919734e-10,
151
+ "model.layers.9.self_attn.v_proj.weight": 2.956998343961459e-09,
152
+ "model.layers.9.self_attn.o_proj.weight": 2.4913035190365917e-09,
153
+ "model.layers.9.mlp.gate_proj.weight": 7.570661182398908e-09,
154
+ "model.layers.9.mlp.up_proj.weight": 6.353935380805827e-09,
155
+ "model.layers.9.mlp.down_proj.weight": 8.711306046886118e-10,
156
+ "model.layers.9.input_layernorm.weight": 0.0,
157
+ "model.layers.9.input_layernorm.bias": 2.168404344971009e-19,
158
+ "model.layers.9.post_attention_layernorm.weight": 0.0,
159
+ "model.layers.9.post_attention_layernorm.bias": 4.906014830496908e-19,
160
+ "model.layers.10.self_attn.q_proj.weight": 5.71690890183163e-10,
161
+ "model.layers.10.self_attn.k_proj.weight": 7.170714866333583e-10,
162
+ "model.layers.10.self_attn.v_proj.weight": 5.3668185273636486e-09,
163
+ "model.layers.10.self_attn.o_proj.weight": 1.271927203561063e-09,
164
+ "model.layers.10.mlp.gate_proj.weight": 5.268815736296043e-09,
165
+ "model.layers.10.mlp.up_proj.weight": 3.586305865610484e-09,
166
+ "model.layers.10.mlp.down_proj.weight": 5.071385089649993e-10,
167
+ "model.layers.10.input_layernorm.weight": 0.0,
168
+ "model.layers.10.input_layernorm.bias": 0.0,
169
+ "model.layers.10.post_attention_layernorm.weight": 0.0,
170
+ "model.layers.10.post_attention_layernorm.bias": 4.445228907190568e-19,
171
+ "model.layers.11.self_attn.q_proj.weight": 7.266194046451346e-10,
172
+ "model.layers.11.self_attn.k_proj.weight": 6.184255596508592e-10,
173
+ "model.layers.11.self_attn.v_proj.weight": 2.661776079548872e-09,
174
+ "model.layers.11.self_attn.o_proj.weight": 1.118204693284497e-09,
175
+ "model.layers.11.mlp.gate_proj.weight": 4.1432076614000836e-09,
176
+ "model.layers.11.mlp.up_proj.weight": 3.5073454303831873e-09,
177
+ "model.layers.11.mlp.down_proj.weight": 5.212785462542039e-10,
178
+ "model.layers.11.input_layernorm.weight": 0.0,
179
+ "model.layers.11.input_layernorm.bias": 0.0,
180
+ "model.layers.11.post_attention_layernorm.weight": 0.0,
181
+ "model.layers.11.post_attention_layernorm.bias": 2.168404344971009e-19,
182
+ "model.layers.12.self_attn.q_proj.weight": 5.733744146141361e-10,
183
+ "model.layers.12.self_attn.k_proj.weight": 5.487667209536085e-10,
184
+ "model.layers.12.self_attn.v_proj.weight": 4.031819287320104e-09,
185
+ "model.layers.12.self_attn.o_proj.weight": 1.7893317760808713e-09,
186
+ "model.layers.12.mlp.gate_proj.weight": 6.7238200548066134e-09,
187
+ "model.layers.12.mlp.up_proj.weight": 4.333610756172371e-09,
188
+ "model.layers.12.mlp.down_proj.weight": 3.8647418894773487e-10,
189
+ "model.layers.12.input_layernorm.weight": 0.0,
190
+ "model.layers.12.input_layernorm.bias": 1.7347234759768072e-19,
191
+ "model.layers.12.post_attention_layernorm.weight": 0.0,
192
+ "model.layers.12.post_attention_layernorm.bias": 1.7347234759768072e-19,
193
+ "model.layers.13.self_attn.q_proj.weight": 3.6206333930977054e-10,
194
+ "model.layers.13.self_attn.k_proj.weight": 3.510607982093461e-10,
195
+ "model.layers.13.self_attn.v_proj.weight": 3.862275548272009e-09,
196
+ "model.layers.13.self_attn.o_proj.weight": 8.800259365671081e-10,
197
+ "model.layers.13.mlp.gate_proj.weight": 5.46204698063472e-09,
198
+ "model.layers.13.mlp.up_proj.weight": 4.405306223513131e-09,
199
+ "model.layers.13.mlp.down_proj.weight": 4.107201808286239e-10,
200
+ "model.layers.13.input_layernorm.weight": 0.0,
201
+ "model.layers.13.input_layernorm.bias": 0.0,
202
+ "model.layers.13.post_attention_layernorm.weight": 0.0,
203
+ "model.layers.13.post_attention_layernorm.bias": 0.0,
204
+ "model.layers.14.self_attn.q_proj.weight": 3.8464165186269383e-10,
205
+ "model.layers.14.self_attn.k_proj.weight": 2.8596257806157154e-10,
206
+ "model.layers.14.self_attn.v_proj.weight": 3.1012530143925687e-09,
207
+ "model.layers.14.self_attn.o_proj.weight": 1.4942531834094552e-09,
208
+ "model.layers.14.mlp.gate_proj.weight": 6.4883253495888005e-09,
209
+ "model.layers.14.mlp.up_proj.weight": 6.394475349225104e-09,
210
+ "model.layers.14.mlp.down_proj.weight": 3.752011915973223e-10,
211
+ "model.layers.14.input_layernorm.weight": 0.0,
212
+ "model.layers.14.input_layernorm.bias": 0.0,
213
+ "model.layers.14.post_attention_layernorm.weight": 0.0,
214
+ "model.layers.14.post_attention_layernorm.bias": 4.0115480381963666e-19,
215
+ "model.layers.15.self_attn.q_proj.weight": 4.776023843078292e-10,
216
+ "model.layers.15.self_attn.k_proj.weight": 4.016206744950068e-10,
217
+ "model.layers.15.self_attn.v_proj.weight": 4.840656515625596e-09,
218
+ "model.layers.15.self_attn.o_proj.weight": 1.2165354235094129e-09,
219
+ "model.layers.15.mlp.gate_proj.weight": 6.1591943752622074e-09,
220
+ "model.layers.15.mlp.up_proj.weight": 5.179849722480867e-09,
221
+ "model.layers.15.mlp.down_proj.weight": 2.731740285829195e-10,
222
+ "model.layers.15.input_layernorm.weight": 0.0,
223
+ "model.layers.15.input_layernorm.bias": 9.75781955236954e-20,
224
+ "model.layers.15.post_attention_layernorm.weight": 0.0,
225
+ "model.layers.15.post_attention_layernorm.bias": 1.951563910473908e-19,
226
+ "model.layers.16.self_attn.q_proj.weight": 2.842091184618312e-10,
227
+ "model.layers.16.self_attn.k_proj.weight": 5.433628658124689e-10,
228
+ "model.layers.16.self_attn.v_proj.weight": 4.038230372316321e-09,
229
+ "model.layers.16.self_attn.o_proj.weight": 8.410473384401484e-10,
230
+ "model.layers.16.mlp.gate_proj.weight": 5.810029506392311e-09,
231
+ "model.layers.16.mlp.up_proj.weight": 4.040571184305008e-09,
232
+ "model.layers.16.mlp.down_proj.weight": 3.436520283154702e-10,
233
+ "model.layers.16.input_layernorm.weight": 0.0,
234
+ "model.layers.16.input_layernorm.bias": 4.336808689942018e-20,
235
+ "model.layers.16.post_attention_layernorm.weight": 0.0,
236
+ "model.layers.16.post_attention_layernorm.bias": 3.4694469519536144e-19,
237
+ "model.layers.17.self_attn.q_proj.weight": 5.428772098525769e-10,
238
+ "model.layers.17.self_attn.k_proj.weight": 4.891210991786466e-10,
239
+ "model.layers.17.self_attn.v_proj.weight": 5.218405476625776e-09,
240
+ "model.layers.17.self_attn.o_proj.weight": 1.6160302607204358e-09,
241
+ "model.layers.17.mlp.gate_proj.weight": 4.256941489681291e-09,
242
+ "model.layers.17.mlp.up_proj.weight": 2.7025096945484016e-09,
243
+ "model.layers.17.mlp.down_proj.weight": 3.0956850120370897e-10,
244
+ "model.layers.17.input_layernorm.weight": 0.0,
245
+ "model.layers.17.input_layernorm.bias": 0.0,
246
+ "model.layers.17.post_attention_layernorm.weight": 0.0,
247
+ "model.layers.17.post_attention_layernorm.bias": 2.2768245622195594e-19,
248
+ "model.layers.18.self_attn.q_proj.weight": 9.754786134408278e-10,
249
+ "model.layers.18.self_attn.k_proj.weight": 1.0905268510441602e-09,
250
+ "model.layers.18.self_attn.v_proj.weight": 1.7681153963167162e-09,
251
+ "model.layers.18.self_attn.o_proj.weight": 6.709172595265045e-10,
252
+ "model.layers.18.mlp.gate_proj.weight": 2.5347956504144045e-09,
253
+ "model.layers.18.mlp.up_proj.weight": 3.966439786988e-09,
254
+ "model.layers.18.mlp.down_proj.weight": 3.36461501622883e-10,
255
+ "model.layers.18.input_layernorm.weight": 0.0,
256
+ "model.layers.18.input_layernorm.bias": 1.7347234759768072e-19,
257
+ "model.layers.18.post_attention_layernorm.weight": 0.0,
258
+ "model.layers.18.post_attention_layernorm.bias": 1.0842021724855045e-20,
259
+ "model.layers.19.self_attn.q_proj.weight": 5.481194875756046e-10,
260
+ "model.layers.19.self_attn.k_proj.weight": 7.619642872214172e-10,
261
+ "model.layers.19.self_attn.v_proj.weight": 2.641356218191504e-09,
262
+ "model.layers.19.self_attn.o_proj.weight": 9.716190874087261e-10,
263
+ "model.layers.19.mlp.gate_proj.weight": 2.508716079319129e-09,
264
+ "model.layers.19.mlp.up_proj.weight": 2.292312615281844e-09,
265
+ "model.layers.19.mlp.down_proj.weight": 3.7359500974313657e-10,
266
+ "model.layers.19.input_layernorm.weight": 0.0,
267
+ "model.layers.19.input_layernorm.bias": 4.336808689942018e-20,
268
+ "model.layers.19.post_attention_layernorm.weight": 0.0,
269
+ "model.layers.19.post_attention_layernorm.bias": 3.496552006265752e-19,
270
+ "model.layers.20.self_attn.q_proj.weight": 4.460271441075747e-10,
271
+ "model.layers.20.self_attn.k_proj.weight": 5.93622040412356e-10,
272
+ "model.layers.20.self_attn.v_proj.weight": 2.9228209541543036e-09,
273
+ "model.layers.20.self_attn.o_proj.weight": 9.93879041288892e-10,
274
+ "model.layers.20.mlp.gate_proj.weight": 4.262360941235481e-09,
275
+ "model.layers.20.mlp.up_proj.weight": 2.8738960130188692e-09,
276
+ "model.layers.20.mlp.down_proj.weight": 3.880129639810548e-10,
277
+ "model.layers.20.input_layernorm.weight": 0.0,
278
+ "model.layers.20.input_layernorm.bias": 2.710505431213761e-19,
279
+ "model.layers.20.post_attention_layernorm.weight": 0.0,
280
+ "model.layers.20.post_attention_layernorm.bias": 3.686287386450715e-19,
281
+ "model.layers.21.self_attn.q_proj.weight": 7.124691592252929e-10,
282
+ "model.layers.21.self_attn.k_proj.weight": 5.755662257911353e-10,
283
+ "model.layers.21.self_attn.v_proj.weight": 1.891308869517161e-09,
284
+ "model.layers.21.self_attn.o_proj.weight": 1.4094457867486198e-09,
285
+ "model.layers.21.mlp.gate_proj.weight": 4.1152960269149236e-09,
286
+ "model.layers.21.mlp.up_proj.weight": 4.749576116106861e-09,
287
+ "model.layers.21.mlp.down_proj.weight": 6.98995326805137e-10,
288
+ "model.layers.21.input_layernorm.weight": 0.0,
289
+ "model.layers.21.input_layernorm.bias": 0.0,
290
+ "model.layers.21.post_attention_layernorm.weight": 0.0,
291
+ "model.layers.21.post_attention_layernorm.bias": 0.0,
292
+ "model.layers.22.self_attn.q_proj.weight": 9.83956525146823e-10,
293
+ "model.layers.22.self_attn.k_proj.weight": 7.087214726197998e-10,
294
+ "model.layers.22.self_attn.v_proj.weight": 2.4996680281219597e-09,
295
+ "model.layers.22.self_attn.o_proj.weight": 8.651306160345484e-10,
296
+ "model.layers.22.mlp.gate_proj.weight": 4.86394924337219e-09,
297
+ "model.layers.22.mlp.up_proj.weight": 4.5905771154745404e-09,
298
+ "model.layers.22.mlp.down_proj.weight": 3.95163937128018e-10,
299
+ "model.layers.22.input_layernorm.weight": 0.0,
300
+ "model.layers.22.input_layernorm.bias": 1.7347234759768072e-19,
301
+ "model.layers.22.post_attention_layernorm.weight": 0.0,
302
+ "model.layers.22.post_attention_layernorm.bias": 4.336808689942018e-20,
303
+ "model.layers.23.self_attn.q_proj.weight": 4.918045704016549e-10,
304
+ "model.layers.23.self_attn.k_proj.weight": 8.33777420439219e-10,
305
+ "model.layers.23.self_attn.v_proj.weight": 2.41253161448185e-09,
306
+ "model.layers.23.self_attn.o_proj.weight": 1.0251370241576296e-09,
307
+ "model.layers.23.mlp.gate_proj.weight": 7.800711803914358e-09,
308
+ "model.layers.23.mlp.up_proj.weight": 3.2208852947708994e-09,
309
+ "model.layers.23.mlp.down_proj.weight": 4.422955157679098e-10,
310
+ "model.layers.23.input_layernorm.weight": 0.0,
311
+ "model.layers.23.input_layernorm.bias": 3.9302328752599536e-19,
312
+ "model.layers.23.post_attention_layernorm.weight": 0.0,
313
+ "model.layers.23.post_attention_layernorm.bias": 0.0,
314
+ "model.layers.24.self_attn.q_proj.weight": 5.472487174529306e-10,
315
+ "model.layers.24.self_attn.k_proj.weight": 4.2753729445621503e-10,
316
+ "model.layers.24.self_attn.v_proj.weight": 1.443581112425818e-09,
317
+ "model.layers.24.self_attn.o_proj.weight": 8.208573376577987e-10,
318
+ "model.layers.24.mlp.gate_proj.weight": 2.998762624883966e-09,
319
+ "model.layers.24.mlp.up_proj.weight": 3.6555445603880797e-09,
320
+ "model.layers.24.mlp.down_proj.weight": 5.055675463457495e-10,
321
+ "model.layers.24.input_layernorm.weight": 0.0,
322
+ "model.layers.24.input_layernorm.bias": 0.0,
323
+ "model.layers.24.post_attention_layernorm.weight": 0.0,
324
+ "model.layers.24.post_attention_layernorm.bias": 0.0,
325
+ "model.layers.25.self_attn.q_proj.weight": 1.166278238429186e-09,
326
+ "model.layers.25.self_attn.k_proj.weight": 7.62426850542397e-10,
327
+ "model.layers.25.self_attn.v_proj.weight": 1.1932593224628363e-09,
328
+ "model.layers.25.self_attn.o_proj.weight": 8.748911994871378e-10,
329
+ "model.layers.25.mlp.gate_proj.weight": 5.748820512962994e-09,
330
+ "model.layers.25.mlp.up_proj.weight": 3.9719869467565635e-09,
331
+ "model.layers.25.mlp.down_proj.weight": 5.551282811211422e-10,
332
+ "model.layers.25.input_layernorm.weight": 0.0,
333
+ "model.layers.25.input_layernorm.bias": 4.336808689942018e-20,
334
+ "model.layers.25.post_attention_layernorm.weight": 0.0,
335
+ "model.layers.25.post_attention_layernorm.bias": 1.951563910473908e-19,
336
+ "model.layers.26.self_attn.q_proj.weight": 1.0714719422821872e-09,
337
+ "model.layers.26.self_attn.k_proj.weight": 6.015845599449676e-10,
338
+ "model.layers.26.self_attn.v_proj.weight": 8.382126992501071e-10,
339
+ "model.layers.26.self_attn.o_proj.weight": 6.916177852644978e-10,
340
+ "model.layers.26.mlp.gate_proj.weight": 5.836964949897568e-09,
341
+ "model.layers.26.mlp.up_proj.weight": 2.5183671444513798e-09,
342
+ "model.layers.26.mlp.down_proj.weight": 5.167603944755683e-10,
343
+ "model.layers.26.input_layernorm.weight": 0.0,
344
+ "model.layers.26.input_layernorm.bias": 0.0,
345
+ "model.layers.26.post_attention_layernorm.weight": 0.0,
346
+ "model.layers.26.post_attention_layernorm.bias": 0.0,
347
+ "model.layers.27.self_attn.q_proj.weight": 1.2824335726691061e-09,
348
+ "model.layers.27.self_attn.k_proj.weight": 5.505033939812165e-10,
349
+ "model.layers.27.self_attn.v_proj.weight": 1.0342314027411703e-09,
350
+ "model.layers.27.self_attn.o_proj.weight": 8.070220758327195e-10,
351
+ "model.layers.27.mlp.gate_proj.weight": 9.268433132092468e-09,
352
+ "model.layers.27.mlp.up_proj.weight": 4.086552962689893e-09,
353
+ "model.layers.27.mlp.down_proj.weight": 5.494656344732599e-10,
354
+ "model.layers.27.input_layernorm.weight": 0.0,
355
+ "model.layers.27.input_layernorm.bias": 0.0,
356
+ "model.layers.27.post_attention_layernorm.weight": 0.0,
357
+ "model.layers.27.post_attention_layernorm.bias": 4.336808689942018e-20,
358
+ "model.layers.28.self_attn.q_proj.weight": 1.4427759253976547e-09,
359
+ "model.layers.28.self_attn.k_proj.weight": 8.499876003043027e-10,
360
+ "model.layers.28.self_attn.v_proj.weight": 2.8016114583806485e-09,
361
+ "model.layers.28.self_attn.o_proj.weight": 8.107200244467094e-10,
362
+ "model.layers.28.mlp.gate_proj.weight": 5.27831199785093e-09,
363
+ "model.layers.28.mlp.up_proj.weight": 5.316902994915533e-09,
364
+ "model.layers.28.mlp.down_proj.weight": 5.632971541065975e-10,
365
+ "model.layers.28.input_layernorm.weight": 0.0,
366
+ "model.layers.28.input_layernorm.bias": 3.4846935450041914e-19,
367
+ "model.layers.28.post_attention_layernorm.weight": 0.0,
368
+ "model.layers.28.post_attention_layernorm.bias": 0.0,
369
+ "model.layers.29.self_attn.q_proj.weight": 9.260426736545924e-10,
370
+ "model.layers.29.self_attn.k_proj.weight": 6.835472277089139e-10,
371
+ "model.layers.29.self_attn.v_proj.weight": 9.57920534006007e-10,
372
+ "model.layers.29.self_attn.o_proj.weight": 9.050408777966367e-10,
373
+ "model.layers.29.mlp.gate_proj.weight": 7.60078516274613e-09,
374
+ "model.layers.29.mlp.up_proj.weight": 3.756985430906449e-09,
375
+ "model.layers.29.mlp.down_proj.weight": 4.79516586437967e-10,
376
+ "model.layers.29.input_layernorm.weight": 0.0,
377
+ "model.layers.29.input_layernorm.bias": 0.0,
378
+ "model.layers.29.post_attention_layernorm.weight": 0.0,
379
+ "model.layers.29.post_attention_layernorm.bias": 0.0,
380
+ "model.layers.30.self_attn.q_proj.weight": 1.3636689288887282e-09,
381
+ "model.layers.30.self_attn.k_proj.weight": 1.5714329038019058e-09,
382
+ "model.layers.30.self_attn.v_proj.weight": 1.1758381646131966e-09,
383
+ "model.layers.30.self_attn.o_proj.weight": 9.795578392868266e-10,
384
+ "model.layers.30.mlp.gate_proj.weight": 9.374549942246327e-09,
385
+ "model.layers.30.mlp.up_proj.weight": 5.656590928992955e-09,
386
+ "model.layers.30.mlp.down_proj.weight": 5.065968859222873e-10,
387
+ "model.layers.30.input_layernorm.weight": 0.0,
388
+ "model.layers.30.input_layernorm.bias": 0.0,
389
+ "model.layers.30.post_attention_layernorm.weight": 0.0,
390
+ "model.layers.30.post_attention_layernorm.bias": 0.0,
391
+ "model.layers.31.self_attn.q_proj.weight": 2.28110124567138e-09,
392
+ "model.layers.31.self_attn.k_proj.weight": 1.3092864037389517e-09,
393
+ "model.layers.31.self_attn.v_proj.weight": 1.0963311325440373e-09,
394
+ "model.layers.31.self_attn.o_proj.weight": 9.979353166045257e-10,
395
+ "model.layers.31.mlp.gate_proj.weight": 1.401771745198251e-08,
396
+ "model.layers.31.mlp.up_proj.weight": 4.416380458375594e-09,
397
+ "model.layers.31.mlp.down_proj.weight": 7.472823426724062e-10,
398
+ "model.layers.31.input_layernorm.weight": 0.0,
399
+ "model.layers.31.input_layernorm.bias": 1.7347234759768072e-19,
400
+ "model.layers.31.post_attention_layernorm.weight": 0.0,
401
+ "model.layers.31.post_attention_layernorm.bias": 1.7347234759768072e-19,
402
+ "model.layers.32.self_attn.q_proj.weight": 1.4159746797304251e-09,
403
+ "model.layers.32.self_attn.k_proj.weight": 1.0075619627514242e-09,
404
+ "model.layers.32.self_attn.v_proj.weight": 2.012204163293063e-09,
405
+ "model.layers.32.self_attn.o_proj.weight": 1.2717424624497653e-09,
406
+ "model.layers.32.mlp.gate_proj.weight": 1.0099245173478266e-08,
407
+ "model.layers.32.mlp.up_proj.weight": 9.207220349101893e-09,
408
+ "model.layers.32.mlp.down_proj.weight": 8.522503473310887e-10,
409
+ "model.layers.32.input_layernorm.weight": 0.0,
410
+ "model.layers.32.input_layernorm.bias": 0.0,
411
+ "model.layers.32.post_attention_layernorm.weight": 0.0,
412
+ "model.layers.32.post_attention_layernorm.bias": 0.0,
413
+ "model.layers.33.self_attn.q_proj.weight": 1.7309234578988253e-09,
414
+ "model.layers.33.self_attn.k_proj.weight": 1.6029864013944461e-09,
415
+ "model.layers.33.self_attn.v_proj.weight": 4.697651831975236e-09,
416
+ "model.layers.33.self_attn.o_proj.weight": 1.531523423636827e-09,
417
+ "model.layers.33.mlp.gate_proj.weight": 1.4452407413045876e-08,
418
+ "model.layers.33.mlp.up_proj.weight": 9.082494519437507e-09,
419
+ "model.layers.33.mlp.down_proj.weight": 9.669621666337964e-10,
420
+ "model.layers.33.input_layernorm.weight": 0.0,
421
+ "model.layers.33.input_layernorm.bias": 4.336808689942018e-20,
422
+ "model.layers.33.post_attention_layernorm.weight": 0.0,
423
+ "model.layers.33.post_attention_layernorm.bias": 0.0,
424
+ "model.layers.34.self_attn.q_proj.weight": 2.4314635993505364e-09,
425
+ "model.layers.34.self_attn.k_proj.weight": 1.703990619716933e-09,
426
+ "model.layers.34.self_attn.v_proj.weight": 2.379507009209192e-09,
427
+ "model.layers.34.self_attn.o_proj.weight": 1.3013656996463397e-09,
428
+ "model.layers.34.mlp.gate_proj.weight": 1.7839614277666746e-08,
429
+ "model.layers.34.mlp.up_proj.weight": 8.645561896022022e-09,
430
+ "model.layers.34.mlp.down_proj.weight": 7.185560946254554e-10,
431
+ "model.layers.34.input_layernorm.weight": 0.0,
432
+ "model.layers.34.input_layernorm.bias": 0.0,
433
+ "model.layers.34.post_attention_layernorm.weight": 0.0,
434
+ "model.layers.34.post_attention_layernorm.bias": 0.0,
435
+ "model.layers.35.self_attn.q_proj.weight": 2.9998821560184297e-09,
436
+ "model.layers.35.self_attn.k_proj.weight": 9.999101990842974e-10,
437
+ "model.layers.35.self_attn.v_proj.weight": 3.91424379131422e-09,
438
+ "model.layers.35.self_attn.o_proj.weight": 1.8113006206021964e-09,
439
+ "model.layers.35.mlp.gate_proj.weight": 2.1299209341426224e-08,
440
+ "model.layers.35.mlp.up_proj.weight": 1.5169396950417043e-08,
441
+ "model.layers.35.mlp.down_proj.weight": 2.0139570248526678e-09,
442
+ "model.layers.35.input_layernorm.weight": 0.0,
443
+ "model.layers.35.input_layernorm.bias": 1.7347234759768072e-19,
444
+ "model.layers.35.post_attention_layernorm.weight": 0.0,
445
+ "model.layers.35.post_attention_layernorm.bias": 0.0,
446
+ "model.layers.36.self_attn.q_proj.weight": 2.6555719045973093e-09,
447
+ "model.layers.36.self_attn.k_proj.weight": 1.5296728861358134e-09,
448
+ "model.layers.36.self_attn.v_proj.weight": 6.2084450291877144e-09,
449
+ "model.layers.36.self_attn.o_proj.weight": 1.9505691284393834e-09,
450
+ "model.layers.36.mlp.gate_proj.weight": 1.6187862759882893e-08,
451
+ "model.layers.36.mlp.up_proj.weight": 1.652175493897327e-08,
452
+ "model.layers.36.mlp.down_proj.weight": 1.3528854954832544e-09,
453
+ "model.layers.36.input_layernorm.weight": 0.0,
454
+ "model.layers.36.input_layernorm.bias": 0.0,
455
+ "model.layers.36.post_attention_layernorm.weight": 0.0,
456
+ "model.layers.36.post_attention_layernorm.bias": 1.7347234759768072e-19,
457
+ "model.layers.37.self_attn.q_proj.weight": 2.406661110399e-09,
458
+ "model.layers.37.self_attn.k_proj.weight": 1.7653792383498513e-09,
459
+ "model.layers.37.self_attn.v_proj.weight": 7.7608228821191e-09,
460
+ "model.layers.37.self_attn.o_proj.weight": 2.089813762040649e-09,
461
+ "model.layers.37.mlp.gate_proj.weight": 1.0708132928508955e-08,
462
+ "model.layers.37.mlp.up_proj.weight": 9.600955763744423e-09,
463
+ "model.layers.37.mlp.down_proj.weight": 1.129514638857169e-09,
464
+ "model.layers.37.input_layernorm.weight": 0.0,
465
+ "model.layers.37.input_layernorm.bias": 0.0,
466
+ "model.layers.37.post_attention_layernorm.weight": 0.0,
467
+ "model.layers.37.post_attention_layernorm.bias": 0.0,
468
+ "model.layers.38.self_attn.q_proj.weight": 2.8534219609355205e-09,
469
+ "model.layers.38.self_attn.k_proj.weight": 2.46495147848691e-09,
470
+ "model.layers.38.self_attn.v_proj.weight": 5.143008934282989e-09,
471
+ "model.layers.38.self_attn.o_proj.weight": 2.608180125207582e-09,
472
+ "model.layers.38.mlp.gate_proj.weight": 1.637199299390583e-08,
473
+ "model.layers.38.mlp.up_proj.weight": 7.963470428270133e-09,
474
+ "model.layers.38.mlp.down_proj.weight": 2.3918623052547144e-09,
475
+ "model.layers.38.input_layernorm.weight": 0.0,
476
+ "model.layers.38.input_layernorm.bias": 0.0,
477
+ "model.layers.38.post_attention_layernorm.weight": 0.0,
478
+ "model.layers.38.post_attention_layernorm.bias": 0.0,
479
+ "model.layers.39.self_attn.q_proj.weight": 1.945519016999242e-09,
480
+ "model.layers.39.self_attn.k_proj.weight": 1.2198944432384451e-09,
481
+ "model.layers.39.self_attn.v_proj.weight": 5.185719373912434e-09,
482
+ "model.layers.39.self_attn.o_proj.weight": 8.505105597578222e-09,
483
+ "model.layers.39.mlp.gate_proj.weight": 2.532518919906579e-08,
484
+ "model.layers.39.mlp.up_proj.weight": 1.6030430742830504e-08,
485
+ "model.layers.39.mlp.down_proj.weight": 2.789723036282036e-09,
486
+ "model.layers.39.input_layernorm.weight": 0.0,
487
+ "model.layers.39.input_layernorm.bias": 0.0,
488
+ "model.layers.39.post_attention_layernorm.weight": 0.0,
489
+ "model.layers.39.post_attention_layernorm.bias": 0.0,
490
+ "model.norm.weight": 0.0,
491
+ "model.norm.bias": 0.0,
492
+ "lm_head.weight": 1.5311139267753199e-19
493
+ },
494
+ "module_to_params": {
495
+ "model.embed_tokens.weight": 2.917434234714228e-19,
496
+ "model.layers.0.self_attn": 7.920487519186281e-09,
497
+ "model.layers.0.mlp": 1.1966177832315832e-08,
498
+ "model.layers.0.input_layernorm": 2.710505431213761e-19,
499
+ "model.layers.0.post_attention_layernorm": 6.938893903907229e-19,
500
+ "model.layers.1.self_attn": 2.2341763070699017e-08,
501
+ "model.layers.1.mlp": 4.642010935640428e-09,
502
+ "model.layers.1.input_layernorm": 3.9916427639358903e-19,
503
+ "model.layers.1.post_attention_layernorm": 1.8431436932253575e-19,
504
+ "model.layers.2.self_attn": 9.60992252885262e-09,
505
+ "model.layers.2.mlp": 3.582362779752657e-09,
506
+ "model.layers.2.input_layernorm": 2.9544509200229997e-19,
507
+ "model.layers.2.post_attention_layernorm": 4.336808689942018e-20,
508
+ "model.layers.3.self_attn": 8.925772405632415e-09,
509
+ "model.layers.3.mlp": 5.664624135685396e-08,
510
+ "model.layers.3.input_layernorm": 1.328147661294743e-19,
511
+ "model.layers.3.post_attention_layernorm": 1.9786689647860455e-19,
512
+ "model.layers.4.self_attn": 7.112303066492131e-09,
513
+ "model.layers.4.mlp": 3.493962632698337e-08,
514
+ "model.layers.4.input_layernorm": 4.797594613248357e-19,
515
+ "model.layers.4.post_attention_layernorm": 1.8702487475374952e-19,
516
+ "model.layers.5.self_attn": 6.413531963289643e-09,
517
+ "model.layers.5.mlp": 7.511871444876305e-09,
518
+ "model.layers.5.input_layernorm": 0.0,
519
+ "model.layers.5.post_attention_layernorm": 4.445228907190568e-19,
520
+ "model.layers.6.self_attn": 5.3933323940214e-09,
521
+ "model.layers.6.mlp": 7.4314101065435046e-09,
522
+ "model.layers.6.input_layernorm": 0.0,
523
+ "model.layers.6.post_attention_layernorm": 2.2768245622195594e-19,
524
+ "model.layers.7.self_attn": 6.313910247968123e-09,
525
+ "model.layers.7.mlp": 5.013754484176995e-09,
526
+ "model.layers.7.input_layernorm": 1.7618285302889447e-19,
527
+ "model.layers.7.post_attention_layernorm": 1.2197274440461925e-19,
528
+ "model.layers.8.self_attn": 6.397113736511528e-09,
529
+ "model.layers.8.mlp": 7.55865182592667e-09,
530
+ "model.layers.8.input_layernorm": 4.336808689942018e-20,
531
+ "model.layers.8.post_attention_layernorm": 0.0,
532
+ "model.layers.9.self_attn": 6.690384495300349e-09,
533
+ "model.layers.9.mlp": 1.4795727167893346e-08,
534
+ "model.layers.9.input_layernorm": 2.168404344971009e-19,
535
+ "model.layers.9.post_attention_layernorm": 4.906014830496908e-19,
536
+ "model.layers.10.self_attn": 7.927508107741233e-09,
537
+ "model.layers.10.mlp": 9.362260110871526e-09,
538
+ "model.layers.10.input_layernorm": 0.0,
539
+ "model.layers.10.post_attention_layernorm": 4.445228907190568e-19,
540
+ "model.layers.11.self_attn": 5.125025737129363e-09,
541
+ "model.layers.11.mlp": 8.171831638037475e-09,
542
+ "model.layers.11.input_layernorm": 0.0,
543
+ "model.layers.11.post_attention_layernorm": 2.168404344971009e-19,
544
+ "model.layers.12.self_attn": 6.943292198968721e-09,
545
+ "model.layers.12.mlp": 1.1443904999926721e-08,
546
+ "model.layers.12.input_layernorm": 1.7347234759768072e-19,
547
+ "model.layers.12.post_attention_layernorm": 1.7347234759768072e-19,
548
+ "model.layers.13.self_attn": 5.4554256223582344e-09,
549
+ "model.layers.13.mlp": 1.0278073384976474e-08,
550
+ "model.layers.13.input_layernorm": 0.0,
551
+ "model.layers.13.post_attention_layernorm": 0.0,
552
+ "model.layers.14.self_attn": 5.2661104277262895e-09,
553
+ "model.layers.14.mlp": 1.3258001890411226e-08,
554
+ "model.layers.14.input_layernorm": 0.0,
555
+ "model.layers.14.post_attention_layernorm": 4.0115480381963666e-19,
556
+ "model.layers.15.self_attn": 6.936414997937845e-09,
557
+ "model.layers.15.mlp": 1.1612218126325994e-08,
558
+ "model.layers.15.input_layernorm": 9.75781955236954e-20,
559
+ "model.layers.15.post_attention_layernorm": 1.951563910473908e-19,
560
+ "model.layers.16.self_attn": 5.706849695030769e-09,
561
+ "model.layers.16.mlp": 1.0194252719012789e-08,
562
+ "model.layers.16.input_layernorm": 4.336808689942018e-20,
563
+ "model.layers.16.post_attention_layernorm": 3.4694469519536144e-19,
564
+ "model.layers.17.self_attn": 7.866434046377435e-09,
565
+ "model.layers.17.mlp": 7.269019685433401e-09,
566
+ "model.layers.17.input_layernorm": 0.0,
567
+ "model.layers.17.post_attention_layernorm": 2.2768245622195594e-19,
568
+ "model.layers.18.self_attn": 4.505038120328209e-09,
569
+ "model.layers.18.mlp": 6.837696939025288e-09,
570
+ "model.layers.18.input_layernorm": 1.7347234759768072e-19,
571
+ "model.layers.18.post_attention_layernorm": 1.0842021724855045e-20,
572
+ "model.layers.19.self_attn": 4.923059080397252e-09,
573
+ "model.layers.19.mlp": 5.174623704344109e-09,
574
+ "model.layers.19.input_layernorm": 4.336808689942018e-20,
575
+ "model.layers.19.post_attention_layernorm": 3.496552006265752e-19,
576
+ "model.layers.20.self_attn": 4.9563491799631266e-09,
577
+ "model.layers.20.mlp": 7.524269918235405e-09,
578
+ "model.layers.20.input_layernorm": 2.710505431213761e-19,
579
+ "model.layers.20.post_attention_layernorm": 3.686287386450715e-19,
580
+ "model.layers.21.self_attn": 4.5887900412822085e-09,
581
+ "model.layers.21.mlp": 9.56386746982692e-09,
582
+ "model.layers.21.input_layernorm": 0.0,
583
+ "model.layers.21.post_attention_layernorm": 0.0,
584
+ "model.layers.22.self_attn": 5.057476641923131e-09,
585
+ "model.layers.22.mlp": 9.849690295974748e-09,
586
+ "model.layers.22.input_layernorm": 1.7347234759768072e-19,
587
+ "model.layers.22.post_attention_layernorm": 4.336808689942018e-20,
588
+ "model.layers.23.self_attn": 4.763250629480353e-09,
589
+ "model.layers.23.mlp": 1.1463892614453168e-08,
590
+ "model.layers.23.input_layernorm": 3.9302328752599536e-19,
591
+ "model.layers.23.post_attention_layernorm": 0.0,
592
+ "model.layers.24.self_attn": 3.239224461992763e-09,
593
+ "model.layers.24.mlp": 7.159874731617795e-09,
594
+ "model.layers.24.input_layernorm": 0.0,
595
+ "model.layers.24.post_attention_layernorm": 0.0,
596
+ "model.layers.25.self_attn": 3.996855610921557e-09,
597
+ "model.layers.25.mlp": 1.0275935740840701e-08,
598
+ "model.layers.25.input_layernorm": 4.336808689942018e-20,
599
+ "model.layers.25.post_attention_layernorm": 1.951563910473908e-19,
600
+ "model.layers.26.self_attn": 3.2028869867417595e-09,
601
+ "model.layers.26.mlp": 8.872092488824517e-09,
602
+ "model.layers.26.input_layernorm": 0.0,
603
+ "model.layers.26.post_attention_layernorm": 0.0,
604
+ "model.layers.27.self_attn": 3.6741904452242126e-09,
605
+ "model.layers.27.mlp": 1.3904451729255621e-08,
606
+ "model.layers.27.input_layernorm": 0.0,
607
+ "model.layers.27.post_attention_layernorm": 4.336808689942018e-20,
608
+ "model.layers.28.self_attn": 5.905095008529315e-09,
609
+ "model.layers.28.mlp": 1.115851214687306e-08,
610
+ "model.layers.28.input_layernorm": 3.4846935450041914e-19,
611
+ "model.layers.28.post_attention_layernorm": 0.0,
612
+ "model.layers.29.self_attn": 3.47255131316615e-09,
613
+ "model.layers.29.mlp": 1.1837287180090546e-08,
614
+ "model.layers.29.input_layernorm": 0.0,
615
+ "model.layers.29.post_attention_layernorm": 0.0,
616
+ "model.layers.30.self_attn": 5.090497836590657e-09,
617
+ "model.layers.30.mlp": 1.553773775716157e-08,
618
+ "model.layers.30.input_layernorm": 0.0,
619
+ "model.layers.30.post_attention_layernorm": 0.0,
620
+ "model.layers.31.self_attn": 5.684654098558895e-09,
621
+ "model.layers.31.mlp": 1.9181380253030513e-08,
622
+ "model.layers.31.input_layernorm": 1.7347234759768072e-19,
623
+ "model.layers.31.post_attention_layernorm": 1.7347234759768072e-19,
624
+ "model.layers.32.self_attn": 5.707483268224678e-09,
625
+ "model.layers.32.mlp": 2.0158715869911248e-08,
626
+ "model.layers.32.input_layernorm": 0.0,
627
+ "model.layers.32.post_attention_layernorm": 0.0,
628
+ "model.layers.33.self_attn": 9.563085114905333e-09,
629
+ "model.layers.33.mlp": 2.4501864099117178e-08,
630
+ "model.layers.33.input_layernorm": 4.336808689942018e-20,
631
+ "model.layers.33.post_attention_layernorm": 0.0,
632
+ "model.layers.34.self_attn": 7.816326927923e-09,
633
+ "model.layers.34.mlp": 2.7203732268314224e-08,
634
+ "model.layers.34.input_layernorm": 0.0,
635
+ "model.layers.34.post_attention_layernorm": 0.0,
636
+ "model.layers.35.self_attn": 9.725336767019142e-09,
637
+ "model.layers.35.mlp": 3.848256331669593e-08,
638
+ "model.layers.35.input_layernorm": 1.7347234759768072e-19,
639
+ "model.layers.35.post_attention_layernorm": 0.0,
640
+ "model.layers.36.self_attn": 1.234425894836022e-08,
641
+ "model.layers.36.mlp": 3.406250319433942e-08,
642
+ "model.layers.36.input_layernorm": 0.0,
643
+ "model.layers.36.post_attention_layernorm": 1.7347234759768072e-19,
644
+ "model.layers.37.self_attn": 1.40226769929086e-08,
645
+ "model.layers.37.mlp": 2.1438603331110546e-08,
646
+ "model.layers.37.input_layernorm": 0.0,
647
+ "model.layers.37.post_attention_layernorm": 0.0,
648
+ "model.layers.38.self_attn": 1.3069562498913002e-08,
649
+ "model.layers.38.mlp": 2.6727325727430677e-08,
650
+ "model.layers.38.input_layernorm": 0.0,
651
+ "model.layers.38.post_attention_layernorm": 0.0,
652
+ "model.layers.39.self_attn": 1.6856238431728344e-08,
653
+ "model.layers.39.mlp": 4.414534297817833e-08,
654
+ "model.layers.39.input_layernorm": 0.0,
655
+ "model.layers.39.post_attention_layernorm": 0.0,
656
+ "model.norm.weight": 0.0,
657
+ "model.norm.bias": 0.0,
658
+ "lm_head.weight": 1.5311139267753199e-19
659
+ }
660
+ }
config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "nebchi/Orion-kor-proof",
3
+ "architectures": [
4
+ "OrionForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "auto_map": {
8
+ "AutoConfig": "nebchi/Orion-kor-proof--configuration_orion.OrionConfig",
9
+ "AutoModelForCausalLM": "nebchi/Orion-kor-proof--modeling_orion.OrionForCausalLM"
10
+ },
11
+ "bos_token_id": 1,
12
+ "eos_token_id": 2,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 5120,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 15360,
17
+ "max_position_embeddings": 4096,
18
+ "max_sequence_length": 4096,
19
+ "model_type": "orion",
20
+ "num_attention_heads": 40,
21
+ "num_hidden_layers": 40,
22
+ "num_key_value_heads": 40,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-05,
26
+ "rope_scaling": null,
27
+ "rope_theta": 10000.0,
28
+ "tie_word_embeddings": false,
29
+ "torch_dtype": "bfloat16",
30
+ "transformers_version": "4.40.2",
31
+ "use_cache": false,
32
+ "vocab_size": 84608
33
+ }
generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "do_sample": true,
5
+ "eos_token_id": 2,
6
+ "max_new_tokens": 1024,
7
+ "pad_token_id": 0,
8
+ "repetition_penalty": 1.05,
9
+ "temperature": 0.3,
10
+ "top_k": 5,
11
+ "top_p": 0.9,
12
+ "transformers_version": "4.40.2"
13
+ }
pytorch_model-00001-of-00002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04e6c54c2531419ef11041d2ba8b35a9e2ccd678345e0b9abadd62546e93ca0d
3
+ size 19951723813
pytorch_model-00002-of-00002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd5b2d69627c76a3fdc08b5f9eb3fc285345c8655780f89e5ae09090952c5ed6
3
+ size 9045837523
pytorch_model.bin.index.json ADDED
@@ -0,0 +1,451 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 28997406720
4
+ },
5
+ "weight_map": {
6
+ "lm_head.weight": "pytorch_model-00002-of-00002.bin",
7
+ "model.embed_tokens.weight": "pytorch_model-00001-of-00002.bin",
8
+ "model.layers.0.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
9
+ "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
10
+ "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
11
+ "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
12
+ "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
13
+ "model.layers.0.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
14
+ "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
15
+ "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
16
+ "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
17
+ "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
18
+ "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
19
+ "model.layers.1.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
20
+ "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
21
+ "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
22
+ "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
23
+ "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
24
+ "model.layers.1.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
25
+ "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
26
+ "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
27
+ "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
28
+ "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
29
+ "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
30
+ "model.layers.10.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
31
+ "model.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
32
+ "model.layers.10.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
33
+ "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
34
+ "model.layers.10.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
35
+ "model.layers.10.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
36
+ "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
37
+ "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
38
+ "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
39
+ "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
40
+ "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
41
+ "model.layers.11.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
42
+ "model.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
43
+ "model.layers.11.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
44
+ "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
45
+ "model.layers.11.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
46
+ "model.layers.11.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
47
+ "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
48
+ "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
49
+ "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
50
+ "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
51
+ "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
52
+ "model.layers.12.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
53
+ "model.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
54
+ "model.layers.12.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
55
+ "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
56
+ "model.layers.12.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
57
+ "model.layers.12.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
58
+ "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
59
+ "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
60
+ "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
61
+ "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
62
+ "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
63
+ "model.layers.13.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
64
+ "model.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
65
+ "model.layers.13.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
66
+ "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
67
+ "model.layers.13.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
68
+ "model.layers.13.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
69
+ "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
70
+ "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
71
+ "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
72
+ "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
73
+ "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
74
+ "model.layers.14.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
75
+ "model.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
76
+ "model.layers.14.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
77
+ "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
78
+ "model.layers.14.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
79
+ "model.layers.14.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
80
+ "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
81
+ "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
82
+ "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
83
+ "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
84
+ "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
85
+ "model.layers.15.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
86
+ "model.layers.15.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
87
+ "model.layers.15.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
88
+ "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
89
+ "model.layers.15.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
90
+ "model.layers.15.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
91
+ "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
92
+ "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
93
+ "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
94
+ "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
95
+ "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
96
+ "model.layers.16.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
97
+ "model.layers.16.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
98
+ "model.layers.16.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
99
+ "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
100
+ "model.layers.16.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
101
+ "model.layers.16.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
102
+ "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
103
+ "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
104
+ "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
105
+ "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
106
+ "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
107
+ "model.layers.17.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
108
+ "model.layers.17.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
109
+ "model.layers.17.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
110
+ "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
111
+ "model.layers.17.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
112
+ "model.layers.17.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
113
+ "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
114
+ "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
115
+ "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
116
+ "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
117
+ "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
118
+ "model.layers.18.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
119
+ "model.layers.18.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
120
+ "model.layers.18.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
121
+ "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
122
+ "model.layers.18.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
123
+ "model.layers.18.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
124
+ "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
125
+ "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
126
+ "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
127
+ "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
128
+ "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
129
+ "model.layers.19.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
130
+ "model.layers.19.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
131
+ "model.layers.19.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
132
+ "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
133
+ "model.layers.19.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
134
+ "model.layers.19.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
135
+ "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
136
+ "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
137
+ "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
138
+ "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
139
+ "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
140
+ "model.layers.2.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
141
+ "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
142
+ "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
143
+ "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
144
+ "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
145
+ "model.layers.2.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
146
+ "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
147
+ "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
148
+ "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
149
+ "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
150
+ "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
151
+ "model.layers.20.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
152
+ "model.layers.20.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
153
+ "model.layers.20.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
154
+ "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
155
+ "model.layers.20.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
156
+ "model.layers.20.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
157
+ "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
158
+ "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
159
+ "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
160
+ "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
161
+ "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
162
+ "model.layers.21.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
163
+ "model.layers.21.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
164
+ "model.layers.21.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
165
+ "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
166
+ "model.layers.21.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
167
+ "model.layers.21.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
168
+ "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
169
+ "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
170
+ "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
171
+ "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
172
+ "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
173
+ "model.layers.22.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
174
+ "model.layers.22.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
175
+ "model.layers.22.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
176
+ "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
177
+ "model.layers.22.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
178
+ "model.layers.22.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
179
+ "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
180
+ "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
181
+ "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
182
+ "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
183
+ "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
184
+ "model.layers.23.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
185
+ "model.layers.23.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
186
+ "model.layers.23.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
187
+ "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
188
+ "model.layers.23.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
189
+ "model.layers.23.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
190
+ "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
191
+ "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
192
+ "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
193
+ "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
194
+ "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
195
+ "model.layers.24.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
196
+ "model.layers.24.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
197
+ "model.layers.24.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
198
+ "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
199
+ "model.layers.24.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
200
+ "model.layers.24.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
201
+ "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
202
+ "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
203
+ "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
204
+ "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
205
+ "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
206
+ "model.layers.25.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
207
+ "model.layers.25.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
208
+ "model.layers.25.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
209
+ "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
210
+ "model.layers.25.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
211
+ "model.layers.25.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
212
+ "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
213
+ "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
214
+ "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
215
+ "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
216
+ "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
217
+ "model.layers.26.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
218
+ "model.layers.26.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
219
+ "model.layers.26.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
220
+ "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
221
+ "model.layers.26.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
222
+ "model.layers.26.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
223
+ "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
224
+ "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
225
+ "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
226
+ "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
227
+ "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
228
+ "model.layers.27.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
229
+ "model.layers.27.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
230
+ "model.layers.27.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
231
+ "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
232
+ "model.layers.27.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
233
+ "model.layers.27.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
234
+ "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
235
+ "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
236
+ "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
237
+ "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
238
+ "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
239
+ "model.layers.28.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
240
+ "model.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
241
+ "model.layers.28.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
242
+ "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
243
+ "model.layers.28.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
244
+ "model.layers.28.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
245
+ "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
246
+ "model.layers.28.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
247
+ "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
248
+ "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
249
+ "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
250
+ "model.layers.29.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
251
+ "model.layers.29.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
252
+ "model.layers.29.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
253
+ "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
254
+ "model.layers.29.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
255
+ "model.layers.29.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
256
+ "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
257
+ "model.layers.29.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
258
+ "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
259
+ "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
260
+ "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
261
+ "model.layers.3.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
262
+ "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
263
+ "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
264
+ "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
265
+ "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
266
+ "model.layers.3.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
267
+ "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
268
+ "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
269
+ "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
270
+ "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
271
+ "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
272
+ "model.layers.30.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
273
+ "model.layers.30.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
274
+ "model.layers.30.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
275
+ "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
276
+ "model.layers.30.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
277
+ "model.layers.30.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
278
+ "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
279
+ "model.layers.30.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
280
+ "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
281
+ "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
282
+ "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
283
+ "model.layers.31.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
284
+ "model.layers.31.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
285
+ "model.layers.31.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
286
+ "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
287
+ "model.layers.31.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
288
+ "model.layers.31.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
289
+ "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
290
+ "model.layers.31.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
291
+ "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
292
+ "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
293
+ "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
294
+ "model.layers.32.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
295
+ "model.layers.32.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
296
+ "model.layers.32.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
297
+ "model.layers.32.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
298
+ "model.layers.32.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
299
+ "model.layers.32.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
300
+ "model.layers.32.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
301
+ "model.layers.32.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
302
+ "model.layers.32.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
303
+ "model.layers.32.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
304
+ "model.layers.32.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
305
+ "model.layers.33.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
306
+ "model.layers.33.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
307
+ "model.layers.33.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
308
+ "model.layers.33.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
309
+ "model.layers.33.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
310
+ "model.layers.33.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
311
+ "model.layers.33.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
312
+ "model.layers.33.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
313
+ "model.layers.33.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
314
+ "model.layers.33.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
315
+ "model.layers.33.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
316
+ "model.layers.34.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
317
+ "model.layers.34.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
318
+ "model.layers.34.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
319
+ "model.layers.34.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
320
+ "model.layers.34.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
321
+ "model.layers.34.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
322
+ "model.layers.34.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
323
+ "model.layers.34.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
324
+ "model.layers.34.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
325
+ "model.layers.34.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
326
+ "model.layers.34.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
327
+ "model.layers.35.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
328
+ "model.layers.35.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
329
+ "model.layers.35.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
330
+ "model.layers.35.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
331
+ "model.layers.35.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
332
+ "model.layers.35.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
333
+ "model.layers.35.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
334
+ "model.layers.35.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
335
+ "model.layers.35.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
336
+ "model.layers.35.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
337
+ "model.layers.35.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
338
+ "model.layers.36.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
339
+ "model.layers.36.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
340
+ "model.layers.36.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
341
+ "model.layers.36.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
342
+ "model.layers.36.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
343
+ "model.layers.36.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
344
+ "model.layers.36.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
345
+ "model.layers.36.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
346
+ "model.layers.36.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
347
+ "model.layers.36.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
348
+ "model.layers.36.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
349
+ "model.layers.37.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
350
+ "model.layers.37.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
351
+ "model.layers.37.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
352
+ "model.layers.37.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
353
+ "model.layers.37.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
354
+ "model.layers.37.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
355
+ "model.layers.37.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
356
+ "model.layers.37.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
357
+ "model.layers.37.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
358
+ "model.layers.37.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
359
+ "model.layers.37.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
360
+ "model.layers.38.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
361
+ "model.layers.38.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
362
+ "model.layers.38.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
363
+ "model.layers.38.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
364
+ "model.layers.38.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
365
+ "model.layers.38.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
366
+ "model.layers.38.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
367
+ "model.layers.38.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
368
+ "model.layers.38.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
369
+ "model.layers.38.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
370
+ "model.layers.38.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
371
+ "model.layers.39.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
372
+ "model.layers.39.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
373
+ "model.layers.39.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
374
+ "model.layers.39.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
375
+ "model.layers.39.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
376
+ "model.layers.39.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
377
+ "model.layers.39.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
378
+ "model.layers.39.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
379
+ "model.layers.39.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
380
+ "model.layers.39.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
381
+ "model.layers.39.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
382
+ "model.layers.4.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
383
+ "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
384
+ "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
385
+ "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
386
+ "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
387
+ "model.layers.4.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
388
+ "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
389
+ "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
390
+ "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
391
+ "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
392
+ "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
393
+ "model.layers.5.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
394
+ "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
395
+ "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
396
+ "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
397
+ "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
398
+ "model.layers.5.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
399
+ "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
400
+ "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
401
+ "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
402
+ "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
403
+ "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
404
+ "model.layers.6.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
405
+ "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
406
+ "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
407
+ "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
408
+ "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
409
+ "model.layers.6.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
410
+ "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
411
+ "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
412
+ "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
413
+ "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
414
+ "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
415
+ "model.layers.7.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
416
+ "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
417
+ "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
418
+ "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
419
+ "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
420
+ "model.layers.7.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
421
+ "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
422
+ "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
423
+ "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
424
+ "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
425
+ "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
426
+ "model.layers.8.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
427
+ "model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
428
+ "model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
429
+ "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
430
+ "model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
431
+ "model.layers.8.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
432
+ "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
433
+ "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
434
+ "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
435
+ "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
436
+ "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
437
+ "model.layers.9.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
438
+ "model.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
439
+ "model.layers.9.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
440
+ "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
441
+ "model.layers.9.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
442
+ "model.layers.9.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
443
+ "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
444
+ "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
445
+ "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
446
+ "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
447
+ "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
448
+ "model.norm.bias": "pytorch_model-00002-of-00002.bin",
449
+ "model.norm.weight": "pytorch_model-00002-of-00002.bin"
450
+ }
451
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<unk>",
18
+ "lstrip": false,
19
+ "normalized": true,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": true,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ded43118b7418f56db97a4eed08a5c265c03120158229ddd4fbcc9658241d5f0
3
+ size 1520600
tokenizer_config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": true,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": true,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ }
29
+ },
30
+ "auto_map": {
31
+ "AutoTokenizer": [
32
+ "nebchi/kor-resume-Orion-14B--tokenization_orion.OrionTokenizer",
33
+ null
34
+ ]
35
+ },
36
+ "bos_token": "<s>",
37
+ "chat_template": "{% for message in messages %}{% if loop.first %}{{ bos_token }}{% endif %}{% if message['role'] == 'user' %}{{ 'Human: ' + message['content'] + '\n\nAssistant: ' + eos_token }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token }}{% endif %}{% endfor %}",
38
+ "clean_up_tokenization_spaces": false,
39
+ "eos_token": "</s>",
40
+ "model_max_length": 4096,
41
+ "pad_token": "<unk>",
42
+ "sp_model_kwargs": {},
43
+ "tokenizer_class": "OrionTokenizer",
44
+ "unk_token": "<unk>"
45
+ }